ed
f5d8ea047a
Adds scripts/audit_tier2_leaks.py as defense-in-depth layer 3 (the pre-commit hook is layer 2; OpenCode permission rules are layer 1). The audit scans the main repo's working tree for files matching the forbidden patterns in conductor/tier2/githooks/forbidden-files.txt. Behavior: - Default mode (exit 0): informational report of any leaks found. Useful for manual inspection and pre-commit workflow. - --strict mode (exit 1 if leaks): CI gate. The hook at the commit boundary is the live guard; this is the safety net for any leak that somehow slips through (manual edits, ops mistakes). - --json mode: machine-readable output for CI integration. Detection rules: - "untracked" status: file exists in working tree but is not in HEAD and not in `git ls-files`. Indicates a leak as a new file. - "modified" status: file is in HEAD but the working tree differs. Indicates a leak in progress (tier-2 setup modified a file). - Files that are tracked and unmodified are NOT reported: the main repo legitimately tracks opencode.json, mcp_paths.toml, etc. — the patterns are about CONTENT (modifications by tier-2), not file existence. Skip rules: - .git/, node_modules/, __pycache__/, .venv/, venv/ (ignored dirs) - tests/ (test infrastructure, not user code) - conductor/ (canonical source for tier-2 files; if they're here in a leak, they were committed, not just sitting in working tree) - .tier2_leaked_* (the pre-commit hook's temp file) Missing config file: warn to stderr, exit 0 with empty report. The hook also no-ops in this case; both layers degrade safely. Tests (tests/test_audit_tier2_leaks.py, 13 cases): - Clean tree returns 0 - Each forbidden file type detected (agent, command, opencode.json, mcp_paths.toml) - Non-forbidden files ignored (including legitimate conductor/tier2/agents/tier2-tech-lead.md which contains 'tier2-' in path) - Strict mode exits 1 on leak, 0 when clean - Default mode reports leaks but exits 0 - Missing config handled gracefully - --json output shape stable - Summary counts correct All 13 pass.
186 lines
7.3 KiB
Python
186 lines
7.3 KiB
Python
"""Tests for scripts/audit_tier2_leaks.py.
|
|
|
|
The audit script defends against tier-2 sandbox-only files leaking into
|
|
the main repo's working tree (defense-in-depth: the pre-commit hook
|
|
prevents leaks during commit, the audit catches anything that slips
|
|
through). It scans the working tree and recent commits for files
|
|
matching the forbidden patterns in
|
|
conductor/tier2/githooks/forbidden-files.txt.
|
|
"""
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
AUDIT = Path("scripts/audit_tier2_leaks.py").resolve()
|
|
|
|
|
|
@pytest.fixture
|
|
def repo_root(tmp_path: Path) -> Path:
|
|
"""Create a minimal repo with the audit script's expected layout."""
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
(repo / "conductor" / "tier2" / "githooks").mkdir(parents=True)
|
|
(repo / "conductor" / "tier2" / "githooks" / "forbidden-files.txt").write_text(
|
|
".opencode/agents/tier2-\n"
|
|
".opencode/commands/tier-2-\n"
|
|
"opencode.json\n"
|
|
"mcp_paths.toml\n"
|
|
)
|
|
# Copy the audit script into the repo so it can be invoked by relative path
|
|
audit_dst = repo / "scripts" / "audit_tier2_leaks.py"
|
|
audit_dst.parent.mkdir(parents=True)
|
|
audit_dst.write_bytes(AUDIT.read_bytes())
|
|
return repo
|
|
|
|
|
|
def _run_audit(cwd: Path, *args: str) -> subprocess.CompletedProcess:
|
|
"""Invoke the audit script with --json for machine-readable output."""
|
|
return subprocess.run(
|
|
[sys.executable, "scripts/audit_tier2_leaks.py", "--json", *args],
|
|
cwd=str(cwd),
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
|
|
def _files_block(result: subprocess.CompletedProcess) -> list[dict]:
|
|
"""Parse audit --json output's 'files' block."""
|
|
return json.loads(result.stdout)["files"]
|
|
|
|
|
|
def test_audit_clean_working_tree_returns_zero(repo_root: Path) -> None:
|
|
"""No forbidden files in working tree: audit exits 0 (informational mode)."""
|
|
result = _run_audit(repo_root)
|
|
assert result.returncode == 0, f"unexpected failure: {result.stderr}"
|
|
files = _files_block(result)
|
|
assert files == [], f"unexpected files: {files}"
|
|
|
|
|
|
def test_audit_detects_forbidden_agent_file_in_working_tree(repo_root: Path) -> None:
|
|
"""An untracked .opencode/agents/tier2-*.md file is reported."""
|
|
opencode_dir = repo_root / ".opencode" / "agents"
|
|
opencode_dir.mkdir(parents=True)
|
|
(opencode_dir / "tier2-autonomous.md").write_text("leak\n")
|
|
result = _run_audit(repo_root)
|
|
files = _files_block(result)
|
|
paths = {f["path"] for f in files}
|
|
assert ".opencode/agents/tier2-autonomous.md" in paths
|
|
|
|
|
|
def test_audit_detects_forbidden_command_file_in_working_tree(repo_root: Path) -> None:
|
|
"""An untracked .opencode/commands/tier-2-*.md file is reported."""
|
|
cmd_dir = repo_root / ".opencode" / "commands"
|
|
cmd_dir.mkdir(parents=True)
|
|
(cmd_dir / "tier-2-auto-execute.md").write_text("leak\n")
|
|
result = _run_audit(repo_root)
|
|
paths = {f["path"] for f in _files_block(result)}
|
|
assert ".opencode/commands/tier-2-auto-execute.md" in paths
|
|
|
|
|
|
def test_audit_detects_modified_opencode_json(repo_root: Path) -> None:
|
|
"""A modified opencode.json (added to the working tree) is reported."""
|
|
(repo_root / "opencode.json").write_text('{"tier2-modified": true}\n')
|
|
result = _run_audit(repo_root)
|
|
paths = {f["path"] for f in _files_block(result)}
|
|
assert "opencode.json" in paths
|
|
|
|
|
|
def test_audit_detects_modified_mcp_paths_toml(repo_root: Path) -> None:
|
|
"""A modified mcp_paths.toml is reported."""
|
|
(repo_root / "mcp_paths.toml").write_text('[allowed_paths]\nextra_dirs = ["leaked"]\n')
|
|
result = _run_audit(repo_root)
|
|
paths = {f["path"] for f in _files_block(result)}
|
|
assert "mcp_paths.toml" in paths
|
|
|
|
|
|
def test_audit_ignores_non_forbidden_files(repo_root: Path) -> None:
|
|
"""Files NOT matching any pattern are not reported."""
|
|
(repo_root / "src.py").write_text("print('hi')\n")
|
|
(repo_root / "README.md").write_text("# Hello\n")
|
|
# conductor/tier2/agents/tier2-tech-lead.md is the INTERACTIVE tier-2
|
|
# tech-lead (main repo agent prompt), not the sandbox tier-2-autonomous.
|
|
# It must NOT be flagged even though its path contains 'tier2-'.
|
|
(repo_root / "conductor" / "tier2" / "agents").mkdir(parents=True, exist_ok=True)
|
|
(repo_root / "conductor" / "tier2" / "agents" / "tier2-tech-lead.md").write_text(
|
|
"# interactive tier-2 (allowed)\n"
|
|
)
|
|
result = _run_audit(repo_root)
|
|
assert result.returncode == 0
|
|
files = _files_block(result)
|
|
assert files == [], f"false positives: {files}"
|
|
|
|
|
|
def test_audit_reports_untracked_and_modified_separately(repo_root: Path) -> None:
|
|
"""Untracked forbidden files: status='untracked'. Modified tracked: 'modified'."""
|
|
# untracked case
|
|
(repo_root / ".opencode" / "agents").mkdir(parents=True)
|
|
(repo_root / ".opencode" / "agents" / "tier2-autonomous.md").write_text("a\n")
|
|
result = _run_audit(repo_root)
|
|
files = _files_block(result)
|
|
status_by_path = {f["path"]: f["status"] for f in files}
|
|
assert status_by_path[".opencode/agents/tier2-autonomous.md"] == "untracked"
|
|
|
|
|
|
def test_audit_strict_exits_nonzero_on_leak(repo_root: Path) -> None:
|
|
"""--strict mode: any leak causes exit 1 (CI gate)."""
|
|
(repo_root / ".opencode" / "agents").mkdir(parents=True)
|
|
(repo_root / ".opencode" / "agents" / "tier2-autonomous.md").write_text("leak\n")
|
|
result = _run_audit(repo_root, "--strict")
|
|
assert result.returncode == 1, f"strict mode should fail: {result.returncode}"
|
|
|
|
|
|
def test_audit_strict_exits_zero_when_clean(repo_root: Path) -> None:
|
|
"""--strict mode with clean tree: exit 0."""
|
|
result = _run_audit(repo_root, "--strict")
|
|
assert result.returncode == 0, f"strict mode should pass: {result.returncode}"
|
|
|
|
|
|
def test_audit_default_mode_exits_zero_even_with_leaks(repo_root: Path) -> None:
|
|
"""Default (informational) mode: leaks are reported but exit 0."""
|
|
(repo_root / "opencode.json").write_text('{"leaked": true}\n')
|
|
result = _run_audit(repo_root)
|
|
assert result.returncode == 0, f"informational mode should pass: {result.returncode}"
|
|
# But the leak IS reported in --json output
|
|
files = _files_block(result)
|
|
paths = {f["path"] for f in files}
|
|
assert "opencode.json" in paths
|
|
|
|
|
|
def test_audit_handles_missing_config_gracefully(repo_root: Path) -> None:
|
|
"""If the forbidden-files.txt config is missing, the audit exits 0 with a
|
|
warning. The audit should not crash on missing config (the hook would
|
|
also no-op in this case; both layers degrade safely)."""
|
|
(repo_root / "conductor" / "tier2" / "githooks" / "forbidden-files.txt").unlink()
|
|
result = _run_audit(repo_root)
|
|
assert result.returncode == 0, f"missing config should not fail: {result.stderr}"
|
|
# No files should be reported (nothing to match against)
|
|
assert _files_block(result) == []
|
|
|
|
|
|
def test_audit_human_readable_output_includes_path(repo_root: Path) -> None:
|
|
"""Without --json, the human-readable report mentions the leaked path."""
|
|
(repo_root / ".opencode" / "agents").mkdir(parents=True)
|
|
(repo_root / ".opencode" / "agents" / "tier2-autonomous.md").write_text("leak\n")
|
|
result = subprocess.run(
|
|
[sys.executable, "scripts/audit_tier2_leaks.py"],
|
|
cwd=str(repo_root),
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.returncode == 0
|
|
assert "tier2-autonomous.md" in result.stdout, (
|
|
f"expected path in stdout, got: {result.stdout!r}"
|
|
)
|
|
|
|
|
|
def test_audit_summary_counts(repo_root: Path) -> None:
|
|
"""JSON output includes a 'summary' block with total counts."""
|
|
(repo_root / "opencode.json").write_text("a\n")
|
|
(repo_root / "mcp_paths.toml").write_text("b\n")
|
|
result = _run_audit(repo_root)
|
|
data = json.loads(result.stdout)
|
|
assert "summary" in data
|
|
assert data["summary"]["total"] >= 2 |