add force push support for history compaction · notplants.bsky.social/git-remote-pds@3ab88ab

+137

e2e-tests/playwright-tests/test_oauth_push.py

··· 433 433 finally: 434 434 import shutil 435 435 shutil.rmtree(repo_dir, ignore_errors=True) 436 + 437 + 438 + def test_force_push_after_compaction(test_config, cargo_binary, auth_config_dir, pds_tokens): 439 + """Test force push after compacting (squashing) git history. 440 + 441 + Flow: 442 + 1. Create repo with multiple commits, push normally 443 + 2. Compact history into a single orphan commit (simulates lichen compact) 444 + 3. Normal push should fail (non-fast-forward) 445 + 4. Force push should succeed 446 + 5. Clone back and verify single commit with correct content 447 + """ 448 + handle = test_config["handle"] 449 + debug_dir = cargo_binary["debug_dir"] 450 + 451 + repo_name = f"playwright-force-{int(time.time())}" 452 + 453 + repo_dir = tempfile.mkdtemp(prefix="git-remote-pds-force-test-") 454 + env = os.environ.copy() 455 + env["HOME"] = auth_config_dir 456 + env["PATH"] = debug_dir + ":" + env.get("PATH", "") 457 + env["PDS_URL"] = test_config["pds"] 458 + 459 + try: 460 + # init repo 461 + subprocess.run(["git", "init", "-b", "main"], cwd=repo_dir, check=True, 462 + capture_output=True) 463 + subprocess.run(["git", "config", "user.email", "test@test.com"], cwd=repo_dir, 464 + check=True, capture_output=True) 465 + subprocess.run(["git", "config", "user.name", "Test User"], cwd=repo_dir, 466 + check=True, capture_output=True) 467 + 468 + pds_remote = f"pds://{handle}/{repo_name}" 469 + subprocess.run(["git", "remote", "add", "origin", pds_remote], cwd=repo_dir, 470 + check=True, capture_output=True) 471 + 472 + # create multiple commits and push 473 + for i in range(3): 474 + with open(os.path.join(repo_dir, f"file{i}.txt"), "w") as f: 475 + f.write(f"Content {i}\n") 476 + subprocess.run(["git", "add", "-A"], cwd=repo_dir, check=True, 477 + capture_output=True) 478 + subprocess.run(["git", "commit", "-m", f"commit {i}"], cwd=repo_dir, 479 + check=True, capture_output=True) 480 + 481 + result = subprocess.run( 482 + ["git", "push", "origin", "main"], 483 + cwd=repo_dir, env=env, capture_output=True, text=True, timeout=60, 484 + ) 485 + assert result.returncode == 0, ( 486 + f"Initial push failed:\nstderr: {result.stderr}" 487 + ) 488 + print("Initial push with 3 commits successful") 489 + 490 + # compact: create orphan branch with current tree 491 + subprocess.run(["git", "checkout", "--orphan", "_lichen_compact"], 492 + cwd=repo_dir, check=True, capture_output=True) 493 + subprocess.run(["git", "add", "-A"], cwd=repo_dir, check=True, 494 + capture_output=True) 495 + subprocess.run(["git", "commit", "-m", "compacted history"], 496 + cwd=repo_dir, check=True, capture_output=True) 497 + subprocess.run(["git", "branch", "-D", "main"], cwd=repo_dir, check=True, 498 + capture_output=True) 499 + subprocess.run(["git", "branch", "-m", "main"], cwd=repo_dir, check=True, 500 + capture_output=True) 501 + subprocess.run(["git", "gc", "--aggressive", "--prune=now"], 502 + cwd=repo_dir, check=True, capture_output=True) 503 + 504 + # verify we have exactly 1 commit now 505 + log_result = subprocess.run( 506 + ["git", "log", "--oneline"], 507 + cwd=repo_dir, capture_output=True, text=True, 508 + ) 509 + commit_count = len(log_result.stdout.strip().split("\n")) 510 + assert commit_count == 1, f"Expected 1 commit after compact, got {commit_count}" 511 + print("Compaction successful: 1 commit") 512 + 513 + # normal push should fail (non-fast-forward) 514 + result = subprocess.run( 515 + ["git", "push", "origin", "main"], 516 + cwd=repo_dir, env=env, capture_output=True, text=True, timeout=60, 517 + ) 518 + assert result.returncode != 0, ( 519 + "Normal push after compaction should fail" 520 + ) 521 + assert "non-fast-forward" in result.stderr, ( 522 + f"Expected non-fast-forward error, got:\n{result.stderr}" 523 + ) 524 + print("Normal push correctly rejected (non-fast-forward)") 525 + 526 + # force push should succeed 527 + result = subprocess.run( 528 + ["git", "push", "--force", "origin", "main"], 529 + cwd=repo_dir, env=env, capture_output=True, text=True, timeout=60, 530 + ) 531 + assert result.returncode == 0, ( 532 + f"Force push failed:\nstderr: {result.stderr}" 533 + ) 534 + print("Force push successful") 535 + 536 + # clone back and verify 537 + clone_dir = tempfile.mkdtemp(prefix="git-remote-pds-force-clone-") 538 + clone_env = os.environ.copy() 539 + clone_env["PATH"] = debug_dir + ":" + clone_env.get("PATH", "") 540 + clone_env["PDS_ACCESS_TOKEN"] = pds_tokens["access_jwt"] 541 + clone_env["PDS_DID"] = pds_tokens["did"] 542 + clone_env["PDS_URL"] = test_config["pds"] 543 + 544 + clone_result = subprocess.run( 545 + ["git", "clone", pds_remote, clone_dir], 546 + env=clone_env, capture_output=True, text=True, timeout=60, 547 + ) 548 + assert clone_result.returncode == 0, ( 549 + f"Clone after force push failed:\nstderr: {clone_result.stderr}" 550 + ) 551 + 552 + # verify all 3 files are present (compaction keeps current tree) 553 + for i in range(3): 554 + fpath = os.path.join(clone_dir, f"file{i}.txt") 555 + assert os.path.isfile(fpath), f"file{i}.txt missing after force push clone" 556 + with open(fpath) as f: 557 + assert f"Content {i}" in f.read() 558 + 559 + # verify only 1 commit in the cloned repo 560 + log_result = subprocess.run( 561 + ["git", "log", "--oneline"], 562 + cwd=clone_dir, capture_output=True, text=True, 563 + ) 564 + clone_commits = len(log_result.stdout.strip().split("\n")) 565 + assert clone_commits == 1, ( 566 + f"Expected 1 commit in clone after force push, got {clone_commits}" 567 + ) 568 + print("Force push clone verification successful: 1 commit, all files present") 569 + 570 + finally: 571 + import shutil 572 + shutil.rmtree(repo_dir, ignore_errors=True)

+239 -24

src/push.rs

··· 29 29 /// Reads the remote state record, determines what's new, creates a 30 30 /// bundle of the delta, uploads it, and updates the state record. 31 31 /// On first push (no existing state), creates a full bundle. 32 + /// 33 + /// When `force` is true, the fast-forward check is skipped and the 34 + /// remote bundles array is replaced (rather than appended to), allowing 35 + /// history rewrites like compaction. Unreferenced blobs are garbage 36 + /// collected by the PDS. 32 37 pub async fn push( 33 38 client: &PdsClient, 34 39 did: &str, 35 40 repo_name: &str, 36 41 repo_path: &Path, 42 + force: bool, 37 43 ) -> Result<PushResult, String> { 38 44 // check local branches first (avoids hitting the network for empty repos) 39 45 let local_refs = get_local_refs(repo_path).await?; ··· 55 61 }; 56 62 57 63 // determine if there's anything new to push 58 - if let Some(ref state) = remote_state { 64 + let is_force_rewrite = if let Some(ref state) = remote_state { 59 65 if refs_match(&local_refs, &state.refs) { 60 66 return Ok(PushResult::AlreadyUpToDate); 61 67 } 62 68 63 - // check for non-fast-forward 64 - check_fast_forward(repo_path, &state.refs, &local_refs).await?; 65 - } 69 + if force { 70 + // skip fast-forward check on force push 71 + tracing::info!("force push: skipping fast-forward check"); 72 + // check if remote tips are ancestors of local — if not, we need a full bundle 73 + !is_fast_forward(repo_path, &state.refs, &local_refs).await 74 + } else { 75 + check_fast_forward(repo_path, &state.refs, &local_refs).await?; 76 + false 77 + } 78 + } else { 79 + false 80 + }; 66 81 67 82 // create the bundle 68 - let bundle = match &remote_state { 69 - None => { 70 - // first push — full bundle 71 - tracing::info!("first push to {}, creating full bundle", repo_name); 72 - create_full_bundle(repo_path).await? 73 - } 74 - Some(state) => { 75 - // incremental — bundle since the last known tips 76 - let since_commits: Vec<&str> = state.refs.iter().map(|r| r.sha.as_str()).collect(); 77 - let ref_names: Vec<&str> = local_refs.iter().map(|r| r.name.as_str()).collect(); 83 + let bundle = if remote_state.is_none() || is_force_rewrite { 84 + // first push or force-rewrite — full bundle 85 + if is_force_rewrite { 78 86 tracing::info!( 79 - "incremental push to {}, {} refs since {} prerequisite(s)", 80 - repo_name, 81 - ref_names.len(), 82 - since_commits.len() 87 + "force push to {}, creating full bundle (history rewritten)", 88 + repo_name 83 89 ); 84 - create_incremental_bundle(repo_path, &ref_names, &since_commits).await? 90 + } else { 91 + tracing::info!("first push to {}, creating full bundle", repo_name); 85 92 } 93 + create_full_bundle(repo_path).await? 94 + } else { 95 + // incremental — bundle since the last known tips 96 + let state = remote_state.as_ref().unwrap(); 97 + let since_commits: Vec<&str> = state.refs.iter().map(|r| r.sha.as_str()).collect(); 98 + let ref_names: Vec<&str> = local_refs.iter().map(|r| r.name.as_str()).collect(); 99 + tracing::info!( 100 + "incremental push to {}, {} refs since {} prerequisite(s)", 101 + repo_name, 102 + ref_names.len(), 103 + since_commits.len() 104 + ); 105 + create_incremental_bundle(repo_path, &ref_names, &since_commits).await? 86 106 }; 87 107 88 108 // upload the bundle blob(s), chunking if needed ··· 106 126 created_at: now.clone(), 107 127 }; 108 128 109 - // build updated state 110 - let mut bundles = match &remote_state { 111 - Some(state) => state.bundles.clone(), 112 - None => vec![], 129 + // build updated state — on force rewrite, replace bundles entirely so the 130 + // PDS garbage-collects the now-unreferenced old blobs 131 + let bundles = if is_force_rewrite { 132 + tracing::info!("force push: replacing remote bundle history"); 133 + vec![entry] 134 + } else { 135 + let mut existing = match &remote_state { 136 + Some(state) => state.bundles.clone(), 137 + None => vec![], 138 + }; 139 + existing.push(entry); 140 + existing 113 141 }; 114 - bundles.push(entry); 115 142 116 143 let new_state = RepoState { 117 144 name: Some(repo_name.to_string()), ··· 184 211 true 185 212 } 186 213 214 + /// Returns true if all remote refs are ancestors of the corresponding local refs. 215 + /// 216 + /// Used during force push to decide whether an incremental or full bundle is 217 + /// needed. Does not error — returns false if any ref is non-fast-forward. 218 + async fn is_fast_forward(repo_path: &Path, remote_refs: &[GitRef], local_refs: &[GitRef]) -> bool { 219 + for remote_ref in remote_refs { 220 + let local = match local_refs.iter().find(|r| r.name == remote_ref.name) { 221 + Some(l) => l, 222 + None => return false, 223 + }; 224 + if local.sha == remote_ref.sha { 225 + continue; 226 + } 227 + let output = tokio::process::Command::new("git") 228 + .args(["merge-base", "--is-ancestor", &remote_ref.sha, &local.sha]) 229 + .current_dir(repo_path) 230 + .output() 231 + .await; 232 + match output { 233 + Ok(o) if o.status.success() => continue, 234 + _ => return false, 235 + } 236 + } 237 + true 238 + } 239 + 187 240 /// Checks that each remote ref's SHA is an ancestor of the corresponding local ref. 188 241 /// 189 242 /// This ensures we're only doing fast-forward pushes. Non-fast-forward ··· 283 336 let a: Vec<GitRef> = vec![]; 284 337 let b: Vec<GitRef> = vec![]; 285 338 assert!(refs_match(&a, &b)); 339 + } 340 + 341 + /// Helper: init a git repo in a temp dir with user config. 342 + async fn init_test_repo() -> tempfile::TempDir { 343 + let tmp = tempfile::tempdir().unwrap(); 344 + tokio::process::Command::new("git") 345 + .args(["init", "-b", "main"]) 346 + .current_dir(tmp.path()) 347 + .output() 348 + .await 349 + .unwrap(); 350 + tokio::process::Command::new("git") 351 + .args(["config", "user.email", "test@test.com"]) 352 + .current_dir(tmp.path()) 353 + .output() 354 + .await 355 + .unwrap(); 356 + tokio::process::Command::new("git") 357 + .args(["config", "user.name", "Test"]) 358 + .current_dir(tmp.path()) 359 + .output() 360 + .await 361 + .unwrap(); 362 + tmp 363 + } 364 + 365 + /// Helper: write, stage, and commit a file. 366 + async fn write_and_commit(dir: &std::path::Path, name: &str, content: &str, msg: &str) { 367 + std::fs::write(dir.join(name), content).unwrap(); 368 + tokio::process::Command::new("git") 369 + .args(["add", "-A"]) 370 + .current_dir(dir) 371 + .output() 372 + .await 373 + .unwrap(); 374 + let output = tokio::process::Command::new("git") 375 + .args(["commit", "-m", msg]) 376 + .current_dir(dir) 377 + .output() 378 + .await 379 + .unwrap(); 380 + assert!(output.status.success()); 381 + } 382 + 383 + /// Helper: get the SHA of HEAD. 384 + async fn head_sha(dir: &std::path::Path) -> String { 385 + let output = tokio::process::Command::new("git") 386 + .args(["rev-parse", "HEAD"]) 387 + .current_dir(dir) 388 + .output() 389 + .await 390 + .unwrap(); 391 + String::from_utf8_lossy(&output.stdout).trim().to_string() 392 + } 393 + 394 + #[tokio::test] 395 + async fn is_fast_forward_true_for_ancestor() { 396 + let repo = init_test_repo().await; 397 + write_and_commit(repo.path(), "a.txt", "hello", "first").await; 398 + let sha1 = head_sha(repo.path()).await; 399 + 400 + write_and_commit(repo.path(), "b.txt", "world", "second").await; 401 + let sha2 = head_sha(repo.path()).await; 402 + 403 + // sha1 is ancestor of sha2 — fast forward 404 + let remote = vec![GitRef::new("refs/heads/main", &sha1)]; 405 + let local = vec![GitRef::new("refs/heads/main", &sha2)]; 406 + assert!(is_fast_forward(repo.path(), &remote, &local).await); 407 + } 408 + 409 + #[tokio::test] 410 + async fn is_fast_forward_false_after_orphan() { 411 + let repo = init_test_repo().await; 412 + write_and_commit(repo.path(), "a.txt", "hello", "first").await; 413 + let old_sha = head_sha(repo.path()).await; 414 + 415 + // simulate compaction: create orphan branch with same content 416 + tokio::process::Command::new("git") 417 + .args(["checkout", "--orphan", "_compact"]) 418 + .current_dir(repo.path()) 419 + .output() 420 + .await 421 + .unwrap(); 422 + tokio::process::Command::new("git") 423 + .args(["commit", "-m", "compacted"]) 424 + .current_dir(repo.path()) 425 + .output() 426 + .await 427 + .unwrap(); 428 + tokio::process::Command::new("git") 429 + .args(["branch", "-D", "main"]) 430 + .current_dir(repo.path()) 431 + .output() 432 + .await 433 + .unwrap(); 434 + tokio::process::Command::new("git") 435 + .args(["branch", "-m", "main"]) 436 + .current_dir(repo.path()) 437 + .output() 438 + .await 439 + .unwrap(); 440 + 441 + let new_sha = head_sha(repo.path()).await; 442 + assert_ne!(old_sha, new_sha); 443 + 444 + // old_sha is NOT an ancestor of new_sha — not fast forward 445 + let remote = vec![GitRef::new("refs/heads/main", &old_sha)]; 446 + let local = vec![GitRef::new("refs/heads/main", &new_sha)]; 447 + assert!(!is_fast_forward(repo.path(), &remote, &local).await); 448 + } 449 + 450 + #[tokio::test] 451 + async fn is_fast_forward_false_for_missing_local_ref() { 452 + let repo = init_test_repo().await; 453 + write_and_commit(repo.path(), "a.txt", "hello", "first").await; 454 + let sha = head_sha(repo.path()).await; 455 + 456 + // remote has a branch that local doesn't 457 + let remote = vec![GitRef::new("refs/heads/other", &sha)]; 458 + let local = vec![GitRef::new("refs/heads/main", &sha)]; 459 + assert!(!is_fast_forward(repo.path(), &remote, &local).await); 460 + } 461 + 462 + #[tokio::test] 463 + async fn check_fast_forward_rejects_non_ff() { 464 + let repo = init_test_repo().await; 465 + write_and_commit(repo.path(), "a.txt", "hello", "first").await; 466 + let old_sha = head_sha(repo.path()).await; 467 + 468 + // create orphan (simulates compaction) 469 + tokio::process::Command::new("git") 470 + .args(["checkout", "--orphan", "_compact"]) 471 + .current_dir(repo.path()) 472 + .output() 473 + .await 474 + .unwrap(); 475 + tokio::process::Command::new("git") 476 + .args(["commit", "-m", "compacted"]) 477 + .current_dir(repo.path()) 478 + .output() 479 + .await 480 + .unwrap(); 481 + tokio::process::Command::new("git") 482 + .args(["branch", "-D", "main"]) 483 + .current_dir(repo.path()) 484 + .output() 485 + .await 486 + .unwrap(); 487 + tokio::process::Command::new("git") 488 + .args(["branch", "-m", "main"]) 489 + .current_dir(repo.path()) 490 + .output() 491 + .await 492 + .unwrap(); 493 + 494 + let new_sha = head_sha(repo.path()).await; 495 + let remote = vec![GitRef::new("refs/heads/main", &old_sha)]; 496 + let local = vec![GitRef::new("refs/heads/main", &new_sha)]; 497 + 498 + let result = check_fast_forward(repo.path(), &remote, &local).await; 499 + assert!(result.is_err()); 500 + assert!(result.unwrap_err().contains("non-fast-forward")); 286 501 } 287 502 }

+11 -2

src/remote_helper.rs

··· 228 228 let repo_path = 229 229 std::env::current_dir().map_err(|e| format!("failed to get current directory: {}", e))?; 230 230 231 - eprintln!("git-remote-pds: pushing to {}/{}", handle, repo_name); 231 + // detect force push: any refspec prefixed with '+' means force 232 + let force = push_lines 233 + .iter() 234 + .any(|l| l.strip_prefix("push ").is_some_and(|r| r.starts_with('+'))); 235 + 236 + if force { 237 + eprintln!("git-remote-pds: force pushing to {}/{}", handle, repo_name); 238 + } else { 239 + eprintln!("git-remote-pds: pushing to {}/{}", handle, repo_name); 240 + } 232 241 233 242 // push using the library 234 - push::push(&client, &auth.did, repo_name, &repo_path).await?; 243 + push::push(&client, &auth.did, repo_name, &repo_path, force).await?; 235 244 236 245 // report ok for each push refspec 237 246 for line in push_lines {

+9 -9

tests/e2e_tests.rs

··· 175 175 176 176 // push to PDS with a unique rkey 177 177 let rkey = unique_rkey("e2e-first-push"); 178 - let result = push(&client, &did, &rkey, repo.path()) 178 + let result = push(&client, &did, &rkey, repo.path(), false) 179 179 .await 180 180 .expect("push failed"); 181 181 ··· 232 232 commit(repo.path(), "first commit").await; 233 233 234 234 let rkey = unique_rkey("e2e-incremental"); 235 - let result = push(&client, &did, &rkey, repo.path()) 235 + let result = push(&client, &did, &rkey, repo.path(), false) 236 236 .await 237 237 .expect("first push failed"); 238 238 assert!( ··· 244 244 write_file(repo.path(), "file2.txt", "second file").await; 245 245 commit(repo.path(), "second commit").await; 246 246 247 - let result = push(&client, &did, &rkey, repo.path()) 247 + let result = push(&client, &did, &rkey, repo.path(), false) 248 248 .await 249 249 .expect("second push failed"); 250 250 assert!( ··· 293 293 commit(repo.path(), "initial").await; 294 294 295 295 let rkey = unique_rkey("e2e-up-to-date"); 296 - push(&client, &did, &rkey, repo.path()) 296 + push(&client, &did, &rkey, repo.path(), false) 297 297 .await 298 298 .expect("first push failed"); 299 299 300 300 // push again with no new commits 301 - let result = push(&client, &did, &rkey, repo.path()) 301 + let result = push(&client, &did, &rkey, repo.path(), false) 302 302 .await 303 303 .expect("second push failed"); 304 304 ··· 368 368 369 369 // push to PDS 370 370 let rkey = unique_rkey("e2e-clone"); 371 - push(&client, &did, &rkey, source.path()) 371 + push(&client, &did, &rkey, source.path(), false) 372 372 .await 373 373 .expect("push failed"); 374 374 ··· 418 418 commit(source.path(), "first").await; 419 419 420 420 let rkey = unique_rkey("e2e-fetch"); 421 - push(&client, &did, &rkey, source.path()) 421 + push(&client, &did, &rkey, source.path(), false) 422 422 .await 423 423 .expect("first push failed"); 424 424 ··· 433 433 commit(source.path(), "second").await; 434 434 let source_sha = head_sha(source.path()).await; 435 435 436 - push(&client, &did, &rkey, source.path()) 436 + push(&client, &did, &rkey, source.path(), false) 437 437 .await 438 438 .expect("second push failed"); 439 439 ··· 653 653 commit(source.path(), "initial").await; 654 654 655 655 let rkey = unique_rkey("e2e-fetch-utd"); 656 - push(&client, &did, &rkey, source.path()) 656 + push(&client, &did, &rkey, source.path(), false) 657 657 .await 658 658 .expect("push failed"); 659 659

+2 -2

tests/push_tests.rs

··· 81 81 // point at a PDS that doesn't exist 82 82 let client = PdsClient::with_auth("http://127.0.0.1:19999", "fake-token"); 83 83 84 - let result = push(&client, "did:plc:test", "test-repo", repo.path()).await; 84 + let result = push(&client, "did:plc:test", "test-repo", repo.path(), false).await; 85 85 assert!(result.is_err()); 86 86 } 87 87 ··· 91 91 let repo = init_repo().await; 92 92 93 93 let client = PdsClient::with_auth("http://127.0.0.1:19999", "fake-token"); 94 - let result = push(&client, "did:plc:test", "test-repo", repo.path()).await; 94 + let result = push(&client, "did:plc:test", "test-repo", repo.path(), false).await; 95 95 assert!(result.is_err()); 96 96 assert!(result.unwrap_err().contains("no local branches")); 97 97 }

Configure Feed

Configure Feed