chore(audit): add audit_test_sandbox_violations.py + 8 regression tests for FR4

2026-06-19 07:26:20 -04:00
parent aa3c993f4a
commit 43e50f9322
2 changed files with 195 additions and 0 deletions
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Detect tests that attempt writes outside ./tests/ via hardcoded paths.
+
+Run from repo root: python scripts/audit_test_sandbox_violations.py
+
+Exit codes:
+  0  CLEAN (or informational mode with violations listed)
+  1  STRICT mode with at least one violation
+
+Patterns flagged:
+  - Path("manual_slop.toml") / Path("config.toml") / etc. (top-level TOML/INI)
+  - open("manual_slop.toml", "w") and similar write-mode calls
+  - Path("C:/projects/...") and Path("C:\\projects\\...") (project root literals)
+  - Path("tests/artifacts/...") literal (violates workspace_paths.md)
+  - tempfile.mkdtemp() / tempfile.mkstemp() without dir= pointing under ./tests/
+
+Reference: conductor/tracks/test_sandbox_hardening_20260619/spec.md (FR4)
+"""
+from __future__ import annotations
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+TOML_BASENAMES = (
+ "manual_slop", "config", "credentials",
+ "presets", "personas", "tool_presets",
+ "workspace_profiles", "project",
+ "manualslop_layout", "manualslop_history",
+)
+INI_BASENAMES = (
+ "manualslop_layout", "manualslop_history",
+)
+_BASENAME_GROUP = "|".join(TOML_BASENAMES)
+_INI_GROUP = "|".join(INI_BASENAMES)
+
+PATTERNS = [
+ re.compile(rf'Path\(["\'](?:{_BASENAME_GROUP})\.toml["\']'),
+ re.compile(rf'Path\(["\'](?:{_INI_GROUP})\.ini["\']'),
+ re.compile(rf'open\(["\'](?:{_BASENAME_GROUP})\.toml["\'], ["\']w["\']'),
+ re.compile(rf'open\(["\'](?:{_BASENAME_GROUP})\.toml["\'], ["\']a["\']'),
+ re.compile(r'Path\(["\']C:[/\\]+projects'),
+ re.compile(r'Path\(["\']tests/artifacts/'),
+ re.compile(r"tempfile\.mk(?:dt|st)emp\("),
+]
+
+EXCLUDE_DIRS = {"artifacts", "logs", "__pycache__", "snapshots"}
+
+
+def find_violations(tests_dir: Path) -> list[tuple[Path, int, str]]:
+ violations: list[tuple[Path, int, str]] = []
+ for test_file in tests_dir.rglob("test_*.py"):
+  if any(excluded in test_file.parts for excluded in EXCLUDE_DIRS):
+   continue
+  try:
+   content = test_file.read_text(encoding="utf-8")
+  except (OSError, UnicodeDecodeError):
+   continue
+  for lineno, line in enumerate(content.splitlines(), start=1):
+   for pattern in PATTERNS:
+    if pattern.search(line):
+     violations.append((test_file, lineno, line.strip()))
+     break
+ return violations
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(
+  description=__doc__,
+  formatter_class=argparse.RawDescriptionHelpFormatter,
+ )
+ parser.add_argument("--json", action="store_true", help="Output JSON instead of human-readable report")
+ parser.add_argument("--strict", action="store_true", help="Exit 1 if any violations are found (CI gate)")
+ parser.add_argument("--tests-dir", default="tests", help="Tests directory to scan (default: tests)")
+ args = parser.parse_args()
+
+ repo_root = Path(__file__).resolve().parent.parent
+ tests_dir = (repo_root / args.tests_dir).resolve() if not Path(args.tests_dir).is_absolute() else Path(args.tests_dir).resolve()
+ if not tests_dir.exists():
+  print(f"Tests dir not found: {tests_dir}", file=sys.stderr)
+  return 1
+ violations = find_violations(tests_dir)
+
+ if args.json:
+  payload = {
+   "tests_dir": str(tests_dir),
+   "count": len(violations),
+   "violations": [
+    {"path": str(p.relative_to(repo_root)), "line": ln, "content": c}
+    for p, ln, c in violations
+   ],
+  }
+  print(json.dumps(payload, indent=2))
+ else:
+  if not violations:
+   print("OK: No test source code references hardcoded paths outside ./tests/.")
+  else:
+   print(f"FAIL: {len(violations)} test source line(s) reference hardcoded paths:")
+   for path, lineno, line in violations:
+    rel = path.relative_to(repo_root)
+    print(f"  {rel}:{lineno}: {line}")
+
+ return 1 if (args.strict and violations) else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
@@ -0,0 +1,87 @@
+"""Tests for scripts/audit_test_sandbox_violations.py (Phase 2, FR4)."""
+from __future__ import annotations
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+def test_audit_runs_without_error() -> None:
+ """The audit script runs and exits cleanly."""
+ result = subprocess.run(
+  [sys.executable, "scripts/audit_test_sandbox_violations.py"],
+  capture_output=True, text=True, cwd=str(Path(__file__).resolve().parent.parent)
+ )
+ assert result.returncode in (0, 1), f"Unexpected exit code: {result.returncode}"
+
+
+def test_audit_flags_toml_basename_pattern() -> None:
+ """A test source line with Path('manual_slop.toml') is flagged by the pattern."""
+ pattern = re.compile(r'Path\(["\'](?:manual_slop|config|credentials|presets|personas|tool_presets|workspace_profiles|project|manualslop_layout|manual_slop_history)\.toml["\']')
+ assert pattern.search('Path("manual_slop.toml").write_text("x")'), "Pattern should match"
+
+
+def test_audit_flags_project_root_path() -> None:
+ """A test source line with Path('C:/projects/...') is flagged."""
+ pattern = re.compile(r'Path\(["\']C:[/\\]+projects')
+ assert pattern.search('base_dir = Path("C:/projects/test")'), "Pattern should match"
+
+
+def test_audit_flags_tempfile_mkdtemp() -> None:
+ """A test source line with bare tempfile.mkdtemp() is flagged."""
+ pattern = re.compile(r"tempfile\.mk(?:dt|st)emp\(")
+ assert pattern.search('tmp = tempfile.mkdtemp()'), "Pattern should match"
+ assert pattern.search('tmp = tempfile.mkstemp()'), "Pattern should match"
+
+
+def test_audit_flags_tests_artifacts_literal() -> None:
+ """A test source line with Path('tests/artifacts/...') literal is flagged."""
+ pattern = re.compile(r'Path\(["\']tests/artifacts/')
+ assert pattern.search('p = Path("tests/artifacts/some_file.txt")'), "Pattern should match"
+
+
+def test_audit_passes_clean_file() -> None:
+ """A test source line using tmp_path passes the audit patterns."""
+ content = 'tmp_path.joinpath("foo.txt").write_text("x")\n'
+ patterns = [
+  re.compile(r'Path\(["\'](?:manual_slop|config)\.toml["\']'),
+  re.compile(r'Path\(["\']C:[/\\]+projects'),
+  re.compile(r'Path\(["\']tests/artifacts/'),
+  re.compile(r"tempfile\.mk(?:dt|st)emp\("),
+ ]
+ for p in patterns:
+  assert not p.search(content), f"Pattern {p.pattern} should not match clean content"
+
+
+def test_audit_subprocess_clean_dir_exits_zero() -> None:
+ """The audit returns 0 on a clean test directory."""
+ tmp_dir = Path("tests/artifacts/_audit_subprocess_clean")
+ tmp_dir.mkdir(parents=True, exist_ok=True)
+ good = tmp_dir / "test_good.py"
+ good.write_text("def test_x(tmp_path): tmp_path.joinpath('f').write_text('x')\n", encoding="utf-8")
+ try:
+  result = subprocess.run(
+   [sys.executable, "scripts/audit_test_sandbox_violations.py", "--tests-dir", str(tmp_dir), "--strict"],
+   capture_output=True, text=True,
+  )
+  assert result.returncode == 0, f"Expected exit 0, got {result.returncode}: {result.stdout}"
+ finally:
+  good.unlink(missing_ok=True)
+  tmp_dir.rmdir()
+
+
+def test_audit_subprocess_bad_dir_exits_one() -> None:
+ """The audit returns 1 on a directory with a bad pattern."""
+ tmp_dir = Path("tests/artifacts/_audit_subprocess_bad")
+ tmp_dir.mkdir(parents=True, exist_ok=True)
+ bad = tmp_dir / "test_bad.py"
+ bad.write_text('Path("manual_slop.toml").write_text("x")\n', encoding="utf-8")
+ try:
+  result = subprocess.run(
+   [sys.executable, "scripts/audit_test_sandbox_violations.py", "--tests-dir", str(tmp_dir), "--strict"],
+   capture_output=True, text=True,
+  )
+  assert result.returncode == 1, f"Expected exit 1, got {result.returncode}"
+ finally:
+  bad.unlink(missing_ok=True)
+  tmp_dir.rmdir()