From 43e50f9322388f79716725ff281abbda5fd68ecc Mon Sep 17 00:00:00 2001 From: Ed_ Date: Fri, 19 Jun 2026 07:26:20 -0400 Subject: [PATCH] chore(audit): add audit_test_sandbox_violations.py + 8 regression tests for FR4 --- scripts/audit_test_sandbox_violations.py | 108 +++++++++++++++++++++++ tests/test_test_sandbox.py | 87 ++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 scripts/audit_test_sandbox_violations.py create mode 100644 tests/test_test_sandbox.py diff --git a/scripts/audit_test_sandbox_violations.py b/scripts/audit_test_sandbox_violations.py new file mode 100644 index 00000000..a5641d97 --- /dev/null +++ b/scripts/audit_test_sandbox_violations.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Detect tests that attempt writes outside ./tests/ via hardcoded paths. + +Run from repo root: python scripts/audit_test_sandbox_violations.py + +Exit codes: + 0 CLEAN (or informational mode with violations listed) + 1 STRICT mode with at least one violation + +Patterns flagged: + - Path("manual_slop.toml") / Path("config.toml") / etc. (top-level TOML/INI) + - open("manual_slop.toml", "w") and similar write-mode calls + - Path("C:/projects/...") and Path("C:\\projects\\...") (project root literals) + - Path("tests/artifacts/...") literal (violates workspace_paths.md) + - tempfile.mkdtemp() / tempfile.mkstemp() without dir= pointing under ./tests/ + +Reference: conductor/tracks/test_sandbox_hardening_20260619/spec.md (FR4) +""" +from __future__ import annotations +import argparse +import json +import re +import sys +from pathlib import Path + +TOML_BASENAMES = ( + "manual_slop", "config", "credentials", + "presets", "personas", "tool_presets", + "workspace_profiles", "project", + "manualslop_layout", "manualslop_history", +) +INI_BASENAMES = ( + "manualslop_layout", "manualslop_history", +) +_BASENAME_GROUP = "|".join(TOML_BASENAMES) +_INI_GROUP = "|".join(INI_BASENAMES) + +PATTERNS = [ + re.compile(rf'Path\(["\'](?:{_BASENAME_GROUP})\.toml["\']'), + re.compile(rf'Path\(["\'](?:{_INI_GROUP})\.ini["\']'), + re.compile(rf'open\(["\'](?:{_BASENAME_GROUP})\.toml["\'], ["\']w["\']'), + re.compile(rf'open\(["\'](?:{_BASENAME_GROUP})\.toml["\'], ["\']a["\']'), + re.compile(r'Path\(["\']C:[/\\]+projects'), + re.compile(r'Path\(["\']tests/artifacts/'), + re.compile(r"tempfile\.mk(?:dt|st)emp\("), +] + +EXCLUDE_DIRS = {"artifacts", "logs", "__pycache__", "snapshots"} + + +def find_violations(tests_dir: Path) -> list[tuple[Path, int, str]]: + violations: list[tuple[Path, int, str]] = [] + for test_file in tests_dir.rglob("test_*.py"): + if any(excluded in test_file.parts for excluded in EXCLUDE_DIRS): + continue + try: + content = test_file.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + for lineno, line in enumerate(content.splitlines(), start=1): + for pattern in PATTERNS: + if pattern.search(line): + violations.append((test_file, lineno, line.strip())) + break + return violations + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--json", action="store_true", help="Output JSON instead of human-readable report") + parser.add_argument("--strict", action="store_true", help="Exit 1 if any violations are found (CI gate)") + parser.add_argument("--tests-dir", default="tests", help="Tests directory to scan (default: tests)") + args = parser.parse_args() + + repo_root = Path(__file__).resolve().parent.parent + tests_dir = (repo_root / args.tests_dir).resolve() if not Path(args.tests_dir).is_absolute() else Path(args.tests_dir).resolve() + if not tests_dir.exists(): + print(f"Tests dir not found: {tests_dir}", file=sys.stderr) + return 1 + violations = find_violations(tests_dir) + + if args.json: + payload = { + "tests_dir": str(tests_dir), + "count": len(violations), + "violations": [ + {"path": str(p.relative_to(repo_root)), "line": ln, "content": c} + for p, ln, c in violations + ], + } + print(json.dumps(payload, indent=2)) + else: + if not violations: + print("OK: No test source code references hardcoded paths outside ./tests/.") + else: + print(f"FAIL: {len(violations)} test source line(s) reference hardcoded paths:") + for path, lineno, line in violations: + rel = path.relative_to(repo_root) + print(f" {rel}:{lineno}: {line}") + + return 1 if (args.strict and violations) else 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/test_test_sandbox.py b/tests/test_test_sandbox.py new file mode 100644 index 00000000..21577f5b --- /dev/null +++ b/tests/test_test_sandbox.py @@ -0,0 +1,87 @@ +"""Tests for scripts/audit_test_sandbox_violations.py (Phase 2, FR4).""" +from __future__ import annotations +import re +import subprocess +import sys +from pathlib import Path + + +def test_audit_runs_without_error() -> None: + """The audit script runs and exits cleanly.""" + result = subprocess.run( + [sys.executable, "scripts/audit_test_sandbox_violations.py"], + capture_output=True, text=True, cwd=str(Path(__file__).resolve().parent.parent) + ) + assert result.returncode in (0, 1), f"Unexpected exit code: {result.returncode}" + + +def test_audit_flags_toml_basename_pattern() -> None: + """A test source line with Path('manual_slop.toml') is flagged by the pattern.""" + pattern = re.compile(r'Path\(["\'](?:manual_slop|config|credentials|presets|personas|tool_presets|workspace_profiles|project|manualslop_layout|manual_slop_history)\.toml["\']') + assert pattern.search('Path("manual_slop.toml").write_text("x")'), "Pattern should match" + + +def test_audit_flags_project_root_path() -> None: + """A test source line with Path('C:/projects/...') is flagged.""" + pattern = re.compile(r'Path\(["\']C:[/\\]+projects') + assert pattern.search('base_dir = Path("C:/projects/test")'), "Pattern should match" + + +def test_audit_flags_tempfile_mkdtemp() -> None: + """A test source line with bare tempfile.mkdtemp() is flagged.""" + pattern = re.compile(r"tempfile\.mk(?:dt|st)emp\(") + assert pattern.search('tmp = tempfile.mkdtemp()'), "Pattern should match" + assert pattern.search('tmp = tempfile.mkstemp()'), "Pattern should match" + + +def test_audit_flags_tests_artifacts_literal() -> None: + """A test source line with Path('tests/artifacts/...') literal is flagged.""" + pattern = re.compile(r'Path\(["\']tests/artifacts/') + assert pattern.search('p = Path("tests/artifacts/some_file.txt")'), "Pattern should match" + + +def test_audit_passes_clean_file() -> None: + """A test source line using tmp_path passes the audit patterns.""" + content = 'tmp_path.joinpath("foo.txt").write_text("x")\n' + patterns = [ + re.compile(r'Path\(["\'](?:manual_slop|config)\.toml["\']'), + re.compile(r'Path\(["\']C:[/\\]+projects'), + re.compile(r'Path\(["\']tests/artifacts/'), + re.compile(r"tempfile\.mk(?:dt|st)emp\("), + ] + for p in patterns: + assert not p.search(content), f"Pattern {p.pattern} should not match clean content" + + +def test_audit_subprocess_clean_dir_exits_zero() -> None: + """The audit returns 0 on a clean test directory.""" + tmp_dir = Path("tests/artifacts/_audit_subprocess_clean") + tmp_dir.mkdir(parents=True, exist_ok=True) + good = tmp_dir / "test_good.py" + good.write_text("def test_x(tmp_path): tmp_path.joinpath('f').write_text('x')\n", encoding="utf-8") + try: + result = subprocess.run( + [sys.executable, "scripts/audit_test_sandbox_violations.py", "--tests-dir", str(tmp_dir), "--strict"], + capture_output=True, text=True, + ) + assert result.returncode == 0, f"Expected exit 0, got {result.returncode}: {result.stdout}" + finally: + good.unlink(missing_ok=True) + tmp_dir.rmdir() + + +def test_audit_subprocess_bad_dir_exits_one() -> None: + """The audit returns 1 on a directory with a bad pattern.""" + tmp_dir = Path("tests/artifacts/_audit_subprocess_bad") + tmp_dir.mkdir(parents=True, exist_ok=True) + bad = tmp_dir / "test_bad.py" + bad.write_text('Path("manual_slop.toml").write_text("x")\n', encoding="utf-8") + try: + result = subprocess.run( + [sys.executable, "scripts/audit_test_sandbox_violations.py", "--tests-dir", str(tmp_dir), "--strict"], + capture_output=True, text=True, + ) + assert result.returncode == 1, f"Expected exit 1, got {result.returncode}" + finally: + bad.unlink(missing_ok=True) + tmp_dir.rmdir() \ No newline at end of file