chore(audit): add audit_test_sandbox_violations.py + 8 regression tests for FR4

2026-06-19 07:26:20 -04:00
parent aa3c993f4a
commit 43e50f9322
2 changed files with 195 additions and 0 deletions
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Detect tests that attempt writes outside ./tests/ via hardcoded paths.
+
+Run from repo root: python scripts/audit_test_sandbox_violations.py
+
+Exit codes:
+  0  CLEAN (or informational mode with violations listed)
+  1  STRICT mode with at least one violation
+
+Patterns flagged:
+  - Path("manual_slop.toml") / Path("config.toml") / etc. (top-level TOML/INI)
+  - open("manual_slop.toml", "w") and similar write-mode calls
+  - Path("C:/projects/...") and Path("C:\\projects\\...") (project root literals)
+  - Path("tests/artifacts/...") literal (violates workspace_paths.md)
+  - tempfile.mkdtemp() / tempfile.mkstemp() without dir= pointing under ./tests/
+
+Reference: conductor/tracks/test_sandbox_hardening_20260619/spec.md (FR4)
+"""
+from __future__ import annotations
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+TOML_BASENAMES = (
+ "manual_slop", "config", "credentials",
+ "presets", "personas", "tool_presets",
+ "workspace_profiles", "project",
+ "manualslop_layout", "manualslop_history",
+)
+INI_BASENAMES = (
+ "manualslop_layout", "manualslop_history",
+)
+_BASENAME_GROUP = "|".join(TOML_BASENAMES)
+_INI_GROUP = "|".join(INI_BASENAMES)
+
+PATTERNS = [
+ re.compile(rf'Path\(["\'](?:{_BASENAME_GROUP})\.toml["\']'),
+ re.compile(rf'Path\(["\'](?:{_INI_GROUP})\.ini["\']'),
+ re.compile(rf'open\(["\'](?:{_BASENAME_GROUP})\.toml["\'], ["\']w["\']'),
+ re.compile(rf'open\(["\'](?:{_BASENAME_GROUP})\.toml["\'], ["\']a["\']'),
+ re.compile(r'Path\(["\']C:[/\\]+projects'),
+ re.compile(r'Path\(["\']tests/artifacts/'),
+ re.compile(r"tempfile\.mk(?:dt|st)emp\("),
+]
+
+EXCLUDE_DIRS = {"artifacts", "logs", "__pycache__", "snapshots"}
+
+
+def find_violations(tests_dir: Path) -> list[tuple[Path, int, str]]:
+ violations: list[tuple[Path, int, str]] = []
+ for test_file in tests_dir.rglob("test_*.py"):
+  if any(excluded in test_file.parts for excluded in EXCLUDE_DIRS):
+   continue
+  try:
+   content = test_file.read_text(encoding="utf-8")
+  except (OSError, UnicodeDecodeError):
+   continue
+  for lineno, line in enumerate(content.splitlines(), start=1):
+   for pattern in PATTERNS:
+    if pattern.search(line):
+     violations.append((test_file, lineno, line.strip()))
+     break
+ return violations
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(
+  description=__doc__,
+  formatter_class=argparse.RawDescriptionHelpFormatter,
+ )
+ parser.add_argument("--json", action="store_true", help="Output JSON instead of human-readable report")
+ parser.add_argument("--strict", action="store_true", help="Exit 1 if any violations are found (CI gate)")
+ parser.add_argument("--tests-dir", default="tests", help="Tests directory to scan (default: tests)")
+ args = parser.parse_args()
+
+ repo_root = Path(__file__).resolve().parent.parent
+ tests_dir = (repo_root / args.tests_dir).resolve() if not Path(args.tests_dir).is_absolute() else Path(args.tests_dir).resolve()
+ if not tests_dir.exists():
+  print(f"Tests dir not found: {tests_dir}", file=sys.stderr)
+  return 1
+ violations = find_violations(tests_dir)
+
+ if args.json:
+  payload = {
+   "tests_dir": str(tests_dir),
+   "count": len(violations),
+   "violations": [
+    {"path": str(p.relative_to(repo_root)), "line": ln, "content": c}
+    for p, ln, c in violations
+   ],
+  }
+  print(json.dumps(payload, indent=2))
+ else:
+  if not violations:
+   print("OK: No test source code references hardcoded paths outside ./tests/.")
+  else:
+   print(f"FAIL: {len(violations)} test source line(s) reference hardcoded paths:")
+   for path, lineno, line in violations:
+    rel = path.relative_to(repo_root)
+    print(f"  {rel}:{lineno}: {line}")
+
+ return 1 if (args.strict and violations) else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())