diff --git a/scripts/audit_tier2_leaks.py b/scripts/audit_tier2_leaks.py new file mode 100644 index 00000000..8a637e69 --- /dev/null +++ b/scripts/audit_tier2_leaks.py @@ -0,0 +1,210 @@ +"""Audit for tier-2 sandbox-only files leaking into the main repo. + +Defense-in-depth layer 3 (after the pre-commit hook at the commit +boundary): scans the working tree for files matching the forbidden +patterns in conductor/tier2/githooks/forbidden-files.txt. If any +match, the file is reported as a leak. + +Usage: + uv run python scripts/audit_tier2_leaks.py # informational + uv run python scripts/audit_tier2_leaks.py --strict # CI gate (exit 1) + uv run python scripts/audit_tier2_leaks.py --json # machine-readable + +Behavior: +- Walks the working tree, skipping .git/, node_modules/, and + __pycache__/ (anything git would ignore at the build level) +- For each candidate file, checks if its relative path contains + any forbidden pattern as a substring +- Reports each leak with its path and status (untracked/modified) +- Default mode exits 0; --strict mode exits 1 if any leaks + +This script is the manual/CI guard. The pre-commit hook at +conductor/tier2/githooks/pre-commit is the live guard; both layers +must be present for the defense-in-depth contract to hold. +""" +import argparse +import json +import subprocess +import sys +from pathlib import Path + +CONFIG_REL = Path("conductor/tier2/githooks/forbidden-files.txt") +SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv"} +# Test infrastructure and the canonical source directory for tier-2 +# files. Tests/ and conductor/tier2/ are project-controlled, not +# tier-2-sandbox-controlled, so the audit ignores them. +SKIP_TOP_DIRS = {"tests", "conductor"} + + +def load_patterns(config_path: Path) -> list[str]: + """Load substring patterns from the denylist config. + + Lines starting with '#' and blank lines are skipped. CR is stripped + (Windows line endings). Each remaining line is a substring to look + for in file paths. + """ + if not config_path.exists(): + return [] + patterns = [] + for raw in config_path.read_text(encoding="utf-8").splitlines(): + line = raw.rstrip("\r") + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + patterns.append(stripped) + return patterns + + +def collect_leaks(repo_root: Path, patterns: list[str]) -> list[dict]: + """Walk the working tree and return files matching any forbidden pattern. + + Each entry: {"path": str (relative), "status": "untracked"|"modified"}. + "modified" = in HEAD but modified in working tree (leak drift in progress). + "untracked" = not in HEAD (a leak staged via git add but not committed yet, + OR a leak as a new untracked file). + + Tracked-but-clean files are NOT reported. The main repo's + opencode.json, mcp_paths.toml, and other tracked forbidden patterns + are legitimate; they are not leaks. Only files that have been + MODIFIED locally (or are NEW) indicate sandbox drift. + """ + if not patterns: + return [] + # Get the set of modified-status from git. This avoids walking + # node_modules and other ignored directories ourselves. + try: + modified_proc = subprocess.run( + ["git", "diff", "--name-only", "-z", "--no-renames"], + cwd=str(repo_root), + capture_output=True, + check=True, + ) + modified = { + p.decode("utf-8") if isinstance(p, bytes) else p + for p in modified_proc.stdout.split(b"\0") + if p + } + except subprocess.CalledProcessError: + modified = set() + + # Get tracked files for the untracked check (a path is untracked iff + # not in `git ls-files`). + try: + tracked_proc = subprocess.run( + ["git", "ls-files", "-z"], + cwd=str(repo_root), + capture_output=True, + check=True, + ) + tracked = { + p.decode("utf-8") if isinstance(p, bytes) else p + for p in tracked_proc.stdout.split(b"\0") + if p + } + except subprocess.CalledProcessError: + tracked = set() + + leaks: list[dict] = [] + # Scan modified files (tracked but changed in working tree) + for rel_path in sorted(modified): + if any(pat in rel_path for pat in patterns): + leaks.append({"path": rel_path, "status": "modified"}) + + # Walk the working tree to catch untracked leaks. We do this manually + # (rather than git ls-files --others --exclude-standard) to keep the + # SKIP_DIRS rules visible in this script. + for path in repo_root.rglob("*"): + if not path.is_file(): + continue + rel = path.relative_to(repo_root).as_posix() + # Skip top-level project directories (tests, conductor) plus the + # standard ignored dirs. + parts = path.relative_to(repo_root).parts + if parts[0] in SKIP_TOP_DIRS: + continue + if any(part in SKIP_DIRS for part in parts): + continue + # Skip the pre-commit hook's temp file + if rel.startswith(".tier2_leaked_"): + continue + if rel in tracked: + continue # already handled above + if any(pat in rel for pat in patterns): + leaks.append({"path": rel, "status": "untracked"}) + + # De-duplicate (in case a path appears in multiple sources) + seen: set[str] = set() + unique: list[dict] = [] + for leak in leaks: + if leak["path"] not in seen: + seen.add(leak["path"]) + unique.append(leak) + return unique + + +def render_human(leaks: list[dict]) -> str: + """Format the leak report for terminal output.""" + if not leaks: + return "[OK] No tier-2 sandbox-only files detected in the working tree.\n" + out = [f"[LEAK] Found {len(leaks)} tier-2 sandbox-only file(s):", ""] + for leak in leaks: + out.append(f" {leak['status']:9s} {leak['path']}") + out.append("") + out.append("These files belong in the main repo only; they are modified by") + out.append("scripts/tier2/setup_tier2_clone.ps1 in the tier-2 clone.") + out.append("If committed, they would absorb the sandbox's local config drift.") + out.append("To remove from the working tree: git rm --cached ") + return "\n".join(out) + "\n" + + +def render_json(leaks: list[dict]) -> str: + """Format the leak report as JSON for machine consumption.""" + return json.dumps( + { + "files": leaks, + "summary": { + "total": len(leaks), + "untracked": sum(1 for l in leaks if l["status"] == "untracked"), + "modified": sum(1 for l in leaks if l["status"] == "modified"), + }, + }, + indent=2, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument( + "--strict", + action="store_true", + help="Exit 1 if any leak is detected. Default: exit 0 (informational).", + ) + parser.add_argument( + "--json", + action="store_true", + help="Emit machine-readable JSON instead of the human-readable report.", + ) + args = parser.parse_args(argv) + + repo_root = Path.cwd() + config_path = repo_root / CONFIG_REL + patterns = load_patterns(config_path) + if not patterns: + print( + f"warning: no forbidden patterns loaded from {config_path}; audit is a no-op.", + file=sys.stderr, + ) + leaks: list[dict] = [] + else: + leaks = collect_leaks(repo_root, patterns) + + if args.json: + print(render_json(leaks)) + else: + print(render_human(leaks), end="") + + return 1 if (args.strict and leaks) else 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/test_audit_tier2_leaks.py b/tests/test_audit_tier2_leaks.py new file mode 100644 index 00000000..283c9306 --- /dev/null +++ b/tests/test_audit_tier2_leaks.py @@ -0,0 +1,186 @@ +"""Tests for scripts/audit_tier2_leaks.py. + +The audit script defends against tier-2 sandbox-only files leaking into +the main repo's working tree (defense-in-depth: the pre-commit hook +prevents leaks during commit, the audit catches anything that slips +through). It scans the working tree and recent commits for files +matching the forbidden patterns in +conductor/tier2/githooks/forbidden-files.txt. +""" +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +AUDIT = Path("scripts/audit_tier2_leaks.py").resolve() + + +@pytest.fixture +def repo_root(tmp_path: Path) -> Path: + """Create a minimal repo with the audit script's expected layout.""" + repo = tmp_path / "repo" + repo.mkdir() + (repo / "conductor" / "tier2" / "githooks").mkdir(parents=True) + (repo / "conductor" / "tier2" / "githooks" / "forbidden-files.txt").write_text( + ".opencode/agents/tier2-\n" + ".opencode/commands/tier-2-\n" + "opencode.json\n" + "mcp_paths.toml\n" + ) + # Copy the audit script into the repo so it can be invoked by relative path + audit_dst = repo / "scripts" / "audit_tier2_leaks.py" + audit_dst.parent.mkdir(parents=True) + audit_dst.write_bytes(AUDIT.read_bytes()) + return repo + + +def _run_audit(cwd: Path, *args: str) -> subprocess.CompletedProcess: + """Invoke the audit script with --json for machine-readable output.""" + return subprocess.run( + [sys.executable, "scripts/audit_tier2_leaks.py", "--json", *args], + cwd=str(cwd), + capture_output=True, + text=True, + ) + + +def _files_block(result: subprocess.CompletedProcess) -> list[dict]: + """Parse audit --json output's 'files' block.""" + return json.loads(result.stdout)["files"] + + +def test_audit_clean_working_tree_returns_zero(repo_root: Path) -> None: + """No forbidden files in working tree: audit exits 0 (informational mode).""" + result = _run_audit(repo_root) + assert result.returncode == 0, f"unexpected failure: {result.stderr}" + files = _files_block(result) + assert files == [], f"unexpected files: {files}" + + +def test_audit_detects_forbidden_agent_file_in_working_tree(repo_root: Path) -> None: + """An untracked .opencode/agents/tier2-*.md file is reported.""" + opencode_dir = repo_root / ".opencode" / "agents" + opencode_dir.mkdir(parents=True) + (opencode_dir / "tier2-autonomous.md").write_text("leak\n") + result = _run_audit(repo_root) + files = _files_block(result) + paths = {f["path"] for f in files} + assert ".opencode/agents/tier2-autonomous.md" in paths + + +def test_audit_detects_forbidden_command_file_in_working_tree(repo_root: Path) -> None: + """An untracked .opencode/commands/tier-2-*.md file is reported.""" + cmd_dir = repo_root / ".opencode" / "commands" + cmd_dir.mkdir(parents=True) + (cmd_dir / "tier-2-auto-execute.md").write_text("leak\n") + result = _run_audit(repo_root) + paths = {f["path"] for f in _files_block(result)} + assert ".opencode/commands/tier-2-auto-execute.md" in paths + + +def test_audit_detects_modified_opencode_json(repo_root: Path) -> None: + """A modified opencode.json (added to the working tree) is reported.""" + (repo_root / "opencode.json").write_text('{"tier2-modified": true}\n') + result = _run_audit(repo_root) + paths = {f["path"] for f in _files_block(result)} + assert "opencode.json" in paths + + +def test_audit_detects_modified_mcp_paths_toml(repo_root: Path) -> None: + """A modified mcp_paths.toml is reported.""" + (repo_root / "mcp_paths.toml").write_text('[allowed_paths]\nextra_dirs = ["leaked"]\n') + result = _run_audit(repo_root) + paths = {f["path"] for f in _files_block(result)} + assert "mcp_paths.toml" in paths + + +def test_audit_ignores_non_forbidden_files(repo_root: Path) -> None: + """Files NOT matching any pattern are not reported.""" + (repo_root / "src.py").write_text("print('hi')\n") + (repo_root / "README.md").write_text("# Hello\n") + # conductor/tier2/agents/tier2-tech-lead.md is the INTERACTIVE tier-2 + # tech-lead (main repo agent prompt), not the sandbox tier-2-autonomous. + # It must NOT be flagged even though its path contains 'tier2-'. + (repo_root / "conductor" / "tier2" / "agents").mkdir(parents=True, exist_ok=True) + (repo_root / "conductor" / "tier2" / "agents" / "tier2-tech-lead.md").write_text( + "# interactive tier-2 (allowed)\n" + ) + result = _run_audit(repo_root) + assert result.returncode == 0 + files = _files_block(result) + assert files == [], f"false positives: {files}" + + +def test_audit_reports_untracked_and_modified_separately(repo_root: Path) -> None: + """Untracked forbidden files: status='untracked'. Modified tracked: 'modified'.""" + # untracked case + (repo_root / ".opencode" / "agents").mkdir(parents=True) + (repo_root / ".opencode" / "agents" / "tier2-autonomous.md").write_text("a\n") + result = _run_audit(repo_root) + files = _files_block(result) + status_by_path = {f["path"]: f["status"] for f in files} + assert status_by_path[".opencode/agents/tier2-autonomous.md"] == "untracked" + + +def test_audit_strict_exits_nonzero_on_leak(repo_root: Path) -> None: + """--strict mode: any leak causes exit 1 (CI gate).""" + (repo_root / ".opencode" / "agents").mkdir(parents=True) + (repo_root / ".opencode" / "agents" / "tier2-autonomous.md").write_text("leak\n") + result = _run_audit(repo_root, "--strict") + assert result.returncode == 1, f"strict mode should fail: {result.returncode}" + + +def test_audit_strict_exits_zero_when_clean(repo_root: Path) -> None: + """--strict mode with clean tree: exit 0.""" + result = _run_audit(repo_root, "--strict") + assert result.returncode == 0, f"strict mode should pass: {result.returncode}" + + +def test_audit_default_mode_exits_zero_even_with_leaks(repo_root: Path) -> None: + """Default (informational) mode: leaks are reported but exit 0.""" + (repo_root / "opencode.json").write_text('{"leaked": true}\n') + result = _run_audit(repo_root) + assert result.returncode == 0, f"informational mode should pass: {result.returncode}" + # But the leak IS reported in --json output + files = _files_block(result) + paths = {f["path"] for f in files} + assert "opencode.json" in paths + + +def test_audit_handles_missing_config_gracefully(repo_root: Path) -> None: + """If the forbidden-files.txt config is missing, the audit exits 0 with a + warning. The audit should not crash on missing config (the hook would + also no-op in this case; both layers degrade safely).""" + (repo_root / "conductor" / "tier2" / "githooks" / "forbidden-files.txt").unlink() + result = _run_audit(repo_root) + assert result.returncode == 0, f"missing config should not fail: {result.stderr}" + # No files should be reported (nothing to match against) + assert _files_block(result) == [] + + +def test_audit_human_readable_output_includes_path(repo_root: Path) -> None: + """Without --json, the human-readable report mentions the leaked path.""" + (repo_root / ".opencode" / "agents").mkdir(parents=True) + (repo_root / ".opencode" / "agents" / "tier2-autonomous.md").write_text("leak\n") + result = subprocess.run( + [sys.executable, "scripts/audit_tier2_leaks.py"], + cwd=str(repo_root), + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert "tier2-autonomous.md" in result.stdout, ( + f"expected path in stdout, got: {result.stdout!r}" + ) + + +def test_audit_summary_counts(repo_root: Path) -> None: + """JSON output includes a 'summary' block with total counts.""" + (repo_root / "opencode.json").write_text("a\n") + (repo_root / "mcp_paths.toml").write_text("b\n") + result = _run_audit(repo_root) + data = json.loads(result.stdout) + assert "summary" in data + assert data["summary"]["total"] >= 2 \ No newline at end of file