solpbc.org / solstone
personal memory agent
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

test: detect leaks into tests/fixtures/ at pytest session end

Add a project-root conftest.py that snapshots `git status --porcelain --
tests/fixtures/` at pytest_sessionstart and diffs at pytest_sessionfinish.
Any new or modified entry relative to the session baseline fails the
session with a named-path stderr message and a 3-5 line guidance block
pointing at journal_copy, the _SOLSTONE_JOURNAL_OVERRIDE env var, and
mocking. Silent on clean sessions; emits a single-line skip warning when
git is unavailable or the working tree is not a repo.

Motivated by two prior fixture-leak incidents:
- f6f382a6 (supervisor schedule tests spawning sol dream and writing
into chronicle/<date>/health/)
- 2996e072 (test_api_baselines per-test autouse clobbering the
isolated override)

Both classes of bug would have been caught at session end by this
detector before landing in a commit.

Covers both pytest entry points (`pytest tests/ --ignore=tests/integration`
from `make test` and `pytest apps/` from `make test-apps`) because pytest
walks from the rootdir (project root, anchored by pyproject.toml) and
loads the top-level conftest.py for both.

Ships with a pytester-based canary at tests/test_fixture_leak_detector.py
that spins up miniature nested pytest sessions in isolated tmp dirs
(with explicit --basetemp to avoid colliding with the repo's
--basetemp=/var/tmp/pytest-solstone) and verifies: (a) the detector
fires and names a leaked path, (b) stays silent on a clean session,
(c) emits the skip warning outside a git repo.

tests/conftest.py's set_test_journal_path default (still tests/fixtures/
journal) is deliberately unchanged — flipping that default is a separate
scope. This lode observes, it does not prevent.

Signed-off-by: Jer Miller <jeremie.miller@gmail.com>

Jer Miller 0fc9ffa6 9ed555db

+220
+92
conftest.py
··· 1 + # SPDX-License-Identifier: AGPL-3.0-only 2 + # Copyright (c) 2026 sol pbc 3 + """Pytest fixture-leak detector. 4 + 5 + Captures `git status --porcelain -- tests/fixtures/` at session start and 6 + diffs at session end. Fails the session with a named-path error when tests 7 + leave the fixture tree dirty. 8 + """ 9 + 10 + from __future__ import annotations 11 + 12 + import subprocess 13 + import sys 14 + from pathlib import Path 15 + 16 + _FIXTURE_ROOT = "tests/fixtures" 17 + _BASELINE: set[tuple[str, str]] | None = None 18 + _GIT_AVAILABLE = True 19 + 20 + 21 + def _capture_status(repo_root: Path) -> set[tuple[str, str]] | None: 22 + """Return the set of (status_XY, path) tuples from fixture-tree git status. 23 + 24 + Returns None when git is unavailable or the command fails (e.g. not a git 25 + repo). 26 + """ 27 + try: 28 + result = subprocess.run( 29 + ["git", "status", "--porcelain", "--", _FIXTURE_ROOT], 30 + cwd=repo_root, 31 + capture_output=True, 32 + text=True, 33 + check=False, 34 + ) 35 + except FileNotFoundError: 36 + return None 37 + if result.returncode != 0: 38 + return None 39 + 40 + entries: set[tuple[str, str]] = set() 41 + for line in result.stdout.splitlines(): 42 + if len(line) < 4: 43 + continue 44 + status = line[:2] 45 + path = line[3:] 46 + if " -> " in path: 47 + path = path.split(" -> ", 1)[1] 48 + entries.add((status, path)) 49 + return entries 50 + 51 + 52 + def _format_leak_message(new_entries: set[tuple[str, str]]) -> str: 53 + lines = [f" {status} {path}" for status, path in sorted(new_entries)] 54 + return ( 55 + "\n" 56 + "solstone fixture-leak detector: tests left tests/fixtures/ dirty\n" 57 + + "\n".join(lines) 58 + + "\n\n" 59 + "To fix, use one of these isolation mechanisms:\n" 60 + " - journal_copy fixture (tests/conftest.py:57) — copies tracked fixtures to tmp_path\n" 61 + " - point _SOLSTONE_JOURNAL_OVERRIDE at a tmp_path directly\n" 62 + " - mock the subprocess/write path so code never touches tests/fixtures/\n" 63 + "\n" 64 + "Prior incidents: f6f382a6, 2996e072\n" 65 + ) 66 + 67 + 68 + def pytest_sessionstart(session): 69 + global _BASELINE, _GIT_AVAILABLE 70 + 71 + repo_root = session.config.rootpath 72 + _BASELINE = _capture_status(repo_root) 73 + if _BASELINE is None: 74 + _GIT_AVAILABLE = False 75 + sys.stderr.write("solstone fixture-leak detector: git unavailable, skipping\n") 76 + 77 + 78 + def pytest_sessionfinish(session, exitstatus): 79 + if not _GIT_AVAILABLE or _BASELINE is None: 80 + return 81 + 82 + repo_root = session.config.rootpath 83 + current = _capture_status(repo_root) 84 + if current is None: 85 + return 86 + 87 + new_entries = current - _BASELINE 88 + if not new_entries: 89 + return 90 + 91 + sys.stderr.write(_format_leak_message(new_entries)) 92 + session.exitstatus = 1
+128
tests/test_fixture_leak_detector.py
··· 1 + # SPDX-License-Identifier: AGPL-3.0-only 2 + # Copyright (c) 2026 sol pbc 3 + """Canary for the fixture-leak detector in the project-root conftest.py.""" 4 + 5 + from __future__ import annotations 6 + 7 + import subprocess 8 + import sys 9 + from pathlib import Path 10 + 11 + import pytest 12 + 13 + pytest_plugins = ["pytester"] 14 + 15 + _ROOT_CONFTEST = Path(__file__).resolve().parent.parent / "conftest.py" 16 + 17 + 18 + def _prime_git_repo(project_dir: Path) -> None: 19 + """Initialise a miniature git repo with one tracked file under tests/fixtures/.""" 20 + subprocess.run(["git", "init", "-q"], cwd=project_dir, check=True) 21 + subprocess.run( 22 + [ 23 + "git", 24 + "-c", 25 + "commit.gpgsign=false", 26 + "-c", 27 + "user.email=canary@example.invalid", 28 + "-c", 29 + "user.name=canary", 30 + "commit", 31 + "--allow-empty", 32 + "-q", 33 + "-m", 34 + "init", 35 + ], 36 + cwd=project_dir, 37 + check=True, 38 + ) 39 + fixtures_dir = project_dir / "tests" / "fixtures" 40 + fixtures_dir.mkdir(parents=True, exist_ok=True) 41 + (fixtures_dir / "keep").write_text("kept\n", encoding="utf-8") 42 + subprocess.run(["git", "add", "tests/fixtures/keep"], cwd=project_dir, check=True) 43 + subprocess.run( 44 + [ 45 + "git", 46 + "-c", 47 + "commit.gpgsign=false", 48 + "-c", 49 + "user.email=canary@example.invalid", 50 + "-c", 51 + "user.name=canary", 52 + "commit", 53 + "-q", 54 + "-m", 55 + "fixtures", 56 + ], 57 + cwd=project_dir, 58 + check=True, 59 + ) 60 + 61 + 62 + def _install_detector(pytester: pytest.Pytester) -> None: 63 + """Copy the real root conftest.py into the pytester project root.""" 64 + pytester.makepyfile(conftest=_ROOT_CONFTEST.read_text(encoding="utf-8")) 65 + 66 + 67 + def _run_nested(pytester: pytest.Pytester) -> pytest.RunResult: 68 + basetemp = pytester.path / "basetemp" 69 + return pytester.run( 70 + sys.executable, 71 + "-mpytest", 72 + "-q", 73 + "-p", 74 + "no:cacheprovider", 75 + "--basetemp", 76 + str(basetemp), 77 + ) 78 + 79 + 80 + @pytest.mark.timeout(30) 81 + def test_detector_fires_on_leaked_file(pytester: pytest.Pytester) -> None: 82 + _install_detector(pytester) 83 + _prime_git_repo(pytester.path) 84 + pytester.makepyfile( 85 + test_leak=""" 86 + from pathlib import Path 87 + 88 + def test_writes_into_fixtures(tmp_path): 89 + Path("tests/fixtures/leak_probe.tmp").write_text("x") 90 + """ 91 + ) 92 + result = _run_nested(pytester) 93 + assert result.ret != 0, result.stderr.str() + result.stdout.str() 94 + combined = result.stderr.str() + result.stdout.str() 95 + assert "solstone fixture-leak detector" in combined 96 + assert "tests/fixtures/leak_probe.tmp" in combined 97 + assert "journal_copy fixture" in combined 98 + 99 + 100 + @pytest.mark.timeout(30) 101 + def test_detector_silent_on_clean(pytester: pytest.Pytester) -> None: 102 + _install_detector(pytester) 103 + _prime_git_repo(pytester.path) 104 + pytester.makepyfile( 105 + test_clean=""" 106 + def test_noop(): 107 + assert True 108 + """ 109 + ) 110 + result = _run_nested(pytester) 111 + assert result.ret == 0, result.stderr.str() + result.stdout.str() 112 + combined = result.stderr.str() + result.stdout.str() 113 + assert "fixture-leak detector" not in combined 114 + 115 + 116 + @pytest.mark.timeout(30) 117 + def test_detector_skips_without_git_repo(pytester: pytest.Pytester) -> None: 118 + _install_detector(pytester) 119 + pytester.makepyfile( 120 + test_clean=""" 121 + def test_noop(): 122 + assert True 123 + """ 124 + ) 125 + result = _run_nested(pytester) 126 + assert result.ret == 0, result.stderr.str() + result.stdout.str() 127 + combined = result.stderr.str() + result.stdout.str() 128 + assert "git unavailable" in combined