feat(audit): add audit_tier2_leaks.py for tier-2 sandbox file leak detection
Adds scripts/audit_tier2_leaks.py as defense-in-depth layer 3 (the pre-commit hook is layer 2; OpenCode permission rules are layer 1). The audit scans the main repo's working tree for files matching the forbidden patterns in conductor/tier2/githooks/forbidden-files.txt. Behavior: - Default mode (exit 0): informational report of any leaks found. Useful for manual inspection and pre-commit workflow. - --strict mode (exit 1 if leaks): CI gate. The hook at the commit boundary is the live guard; this is the safety net for any leak that somehow slips through (manual edits, ops mistakes). - --json mode: machine-readable output for CI integration. Detection rules: - "untracked" status: file exists in working tree but is not in HEAD and not in `git ls-files`. Indicates a leak as a new file. - "modified" status: file is in HEAD but the working tree differs. Indicates a leak in progress (tier-2 setup modified a file). - Files that are tracked and unmodified are NOT reported: the main repo legitimately tracks opencode.json, mcp_paths.toml, etc. — the patterns are about CONTENT (modifications by tier-2), not file existence. Skip rules: - .git/, node_modules/, __pycache__/, .venv/, venv/ (ignored dirs) - tests/ (test infrastructure, not user code) - conductor/ (canonical source for tier-2 files; if they're here in a leak, they were committed, not just sitting in working tree) - .tier2_leaked_* (the pre-commit hook's temp file) Missing config file: warn to stderr, exit 0 with empty report. The hook also no-ops in this case; both layers degrade safely. Tests (tests/test_audit_tier2_leaks.py, 13 cases): - Clean tree returns 0 - Each forbidden file type detected (agent, command, opencode.json, mcp_paths.toml) - Non-forbidden files ignored (including legitimate conductor/tier2/agents/tier2-tech-lead.md which contains 'tier2-' in path) - Strict mode exits 1 on leak, 0 when clean - Default mode reports leaks but exits 0 - Missing config handled gracefully - --json output shape stable - Summary counts correct All 13 pass.
This commit is contained in:
@@ -0,0 +1,210 @@
|
||||
"""Audit for tier-2 sandbox-only files leaking into the main repo.
|
||||
|
||||
Defense-in-depth layer 3 (after the pre-commit hook at the commit
|
||||
boundary): scans the working tree for files matching the forbidden
|
||||
patterns in conductor/tier2/githooks/forbidden-files.txt. If any
|
||||
match, the file is reported as a leak.
|
||||
|
||||
Usage:
|
||||
uv run python scripts/audit_tier2_leaks.py # informational
|
||||
uv run python scripts/audit_tier2_leaks.py --strict # CI gate (exit 1)
|
||||
uv run python scripts/audit_tier2_leaks.py --json # machine-readable
|
||||
|
||||
Behavior:
|
||||
- Walks the working tree, skipping .git/, node_modules/, and
|
||||
__pycache__/ (anything git would ignore at the build level)
|
||||
- For each candidate file, checks if its relative path contains
|
||||
any forbidden pattern as a substring
|
||||
- Reports each leak with its path and status (untracked/modified)
|
||||
- Default mode exits 0; --strict mode exits 1 if any leaks
|
||||
|
||||
This script is the manual/CI guard. The pre-commit hook at
|
||||
conductor/tier2/githooks/pre-commit is the live guard; both layers
|
||||
must be present for the defense-in-depth contract to hold.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
CONFIG_REL = Path("conductor/tier2/githooks/forbidden-files.txt")
|
||||
SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv"}
|
||||
# Test infrastructure and the canonical source directory for tier-2
|
||||
# files. Tests/ and conductor/tier2/ are project-controlled, not
|
||||
# tier-2-sandbox-controlled, so the audit ignores them.
|
||||
SKIP_TOP_DIRS = {"tests", "conductor"}
|
||||
|
||||
|
||||
def load_patterns(config_path: Path) -> list[str]:
|
||||
"""Load substring patterns from the denylist config.
|
||||
|
||||
Lines starting with '#' and blank lines are skipped. CR is stripped
|
||||
(Windows line endings). Each remaining line is a substring to look
|
||||
for in file paths.
|
||||
"""
|
||||
if not config_path.exists():
|
||||
return []
|
||||
patterns = []
|
||||
for raw in config_path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.rstrip("\r")
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
patterns.append(stripped)
|
||||
return patterns
|
||||
|
||||
|
||||
def collect_leaks(repo_root: Path, patterns: list[str]) -> list[dict]:
|
||||
"""Walk the working tree and return files matching any forbidden pattern.
|
||||
|
||||
Each entry: {"path": str (relative), "status": "untracked"|"modified"}.
|
||||
"modified" = in HEAD but modified in working tree (leak drift in progress).
|
||||
"untracked" = not in HEAD (a leak staged via git add but not committed yet,
|
||||
OR a leak as a new untracked file).
|
||||
|
||||
Tracked-but-clean files are NOT reported. The main repo's
|
||||
opencode.json, mcp_paths.toml, and other tracked forbidden patterns
|
||||
are legitimate; they are not leaks. Only files that have been
|
||||
MODIFIED locally (or are NEW) indicate sandbox drift.
|
||||
"""
|
||||
if not patterns:
|
||||
return []
|
||||
# Get the set of modified-status from git. This avoids walking
|
||||
# node_modules and other ignored directories ourselves.
|
||||
try:
|
||||
modified_proc = subprocess.run(
|
||||
["git", "diff", "--name-only", "-z", "--no-renames"],
|
||||
cwd=str(repo_root),
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
modified = {
|
||||
p.decode("utf-8") if isinstance(p, bytes) else p
|
||||
for p in modified_proc.stdout.split(b"\0")
|
||||
if p
|
||||
}
|
||||
except subprocess.CalledProcessError:
|
||||
modified = set()
|
||||
|
||||
# Get tracked files for the untracked check (a path is untracked iff
|
||||
# not in `git ls-files`).
|
||||
try:
|
||||
tracked_proc = subprocess.run(
|
||||
["git", "ls-files", "-z"],
|
||||
cwd=str(repo_root),
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
tracked = {
|
||||
p.decode("utf-8") if isinstance(p, bytes) else p
|
||||
for p in tracked_proc.stdout.split(b"\0")
|
||||
if p
|
||||
}
|
||||
except subprocess.CalledProcessError:
|
||||
tracked = set()
|
||||
|
||||
leaks: list[dict] = []
|
||||
# Scan modified files (tracked but changed in working tree)
|
||||
for rel_path in sorted(modified):
|
||||
if any(pat in rel_path for pat in patterns):
|
||||
leaks.append({"path": rel_path, "status": "modified"})
|
||||
|
||||
# Walk the working tree to catch untracked leaks. We do this manually
|
||||
# (rather than git ls-files --others --exclude-standard) to keep the
|
||||
# SKIP_DIRS rules visible in this script.
|
||||
for path in repo_root.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
rel = path.relative_to(repo_root).as_posix()
|
||||
# Skip top-level project directories (tests, conductor) plus the
|
||||
# standard ignored dirs.
|
||||
parts = path.relative_to(repo_root).parts
|
||||
if parts[0] in SKIP_TOP_DIRS:
|
||||
continue
|
||||
if any(part in SKIP_DIRS for part in parts):
|
||||
continue
|
||||
# Skip the pre-commit hook's temp file
|
||||
if rel.startswith(".tier2_leaked_"):
|
||||
continue
|
||||
if rel in tracked:
|
||||
continue # already handled above
|
||||
if any(pat in rel for pat in patterns):
|
||||
leaks.append({"path": rel, "status": "untracked"})
|
||||
|
||||
# De-duplicate (in case a path appears in multiple sources)
|
||||
seen: set[str] = set()
|
||||
unique: list[dict] = []
|
||||
for leak in leaks:
|
||||
if leak["path"] not in seen:
|
||||
seen.add(leak["path"])
|
||||
unique.append(leak)
|
||||
return unique
|
||||
|
||||
|
||||
def render_human(leaks: list[dict]) -> str:
|
||||
"""Format the leak report for terminal output."""
|
||||
if not leaks:
|
||||
return "[OK] No tier-2 sandbox-only files detected in the working tree.\n"
|
||||
out = [f"[LEAK] Found {len(leaks)} tier-2 sandbox-only file(s):", ""]
|
||||
for leak in leaks:
|
||||
out.append(f" {leak['status']:9s} {leak['path']}")
|
||||
out.append("")
|
||||
out.append("These files belong in the main repo only; they are modified by")
|
||||
out.append("scripts/tier2/setup_tier2_clone.ps1 in the tier-2 clone.")
|
||||
out.append("If committed, they would absorb the sandbox's local config drift.")
|
||||
out.append("To remove from the working tree: git rm --cached <path>")
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
def render_json(leaks: list[dict]) -> str:
|
||||
"""Format the leak report as JSON for machine consumption."""
|
||||
return json.dumps(
|
||||
{
|
||||
"files": leaks,
|
||||
"summary": {
|
||||
"total": len(leaks),
|
||||
"untracked": sum(1 for l in leaks if l["status"] == "untracked"),
|
||||
"modified": sum(1 for l in leaks if l["status"] == "modified"),
|
||||
},
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
parser.add_argument(
|
||||
"--strict",
|
||||
action="store_true",
|
||||
help="Exit 1 if any leak is detected. Default: exit 0 (informational).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
help="Emit machine-readable JSON instead of the human-readable report.",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
repo_root = Path.cwd()
|
||||
config_path = repo_root / CONFIG_REL
|
||||
patterns = load_patterns(config_path)
|
||||
if not patterns:
|
||||
print(
|
||||
f"warning: no forbidden patterns loaded from {config_path}; audit is a no-op.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
leaks: list[dict] = []
|
||||
else:
|
||||
leaks = collect_leaks(repo_root, patterns)
|
||||
|
||||
if args.json:
|
||||
print(render_json(leaks))
|
||||
else:
|
||||
print(render_human(leaks), end="")
|
||||
|
||||
return 1 if (args.strict and leaks) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Tests for scripts/audit_tier2_leaks.py.
|
||||
|
||||
The audit script defends against tier-2 sandbox-only files leaking into
|
||||
the main repo's working tree (defense-in-depth: the pre-commit hook
|
||||
prevents leaks during commit, the audit catches anything that slips
|
||||
through). It scans the working tree and recent commits for files
|
||||
matching the forbidden patterns in
|
||||
conductor/tier2/githooks/forbidden-files.txt.
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
AUDIT = Path("scripts/audit_tier2_leaks.py").resolve()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repo_root(tmp_path: Path) -> Path:
|
||||
"""Create a minimal repo with the audit script's expected layout."""
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "conductor" / "tier2" / "githooks").mkdir(parents=True)
|
||||
(repo / "conductor" / "tier2" / "githooks" / "forbidden-files.txt").write_text(
|
||||
".opencode/agents/tier2-\n"
|
||||
".opencode/commands/tier-2-\n"
|
||||
"opencode.json\n"
|
||||
"mcp_paths.toml\n"
|
||||
)
|
||||
# Copy the audit script into the repo so it can be invoked by relative path
|
||||
audit_dst = repo / "scripts" / "audit_tier2_leaks.py"
|
||||
audit_dst.parent.mkdir(parents=True)
|
||||
audit_dst.write_bytes(AUDIT.read_bytes())
|
||||
return repo
|
||||
|
||||
|
||||
def _run_audit(cwd: Path, *args: str) -> subprocess.CompletedProcess:
|
||||
"""Invoke the audit script with --json for machine-readable output."""
|
||||
return subprocess.run(
|
||||
[sys.executable, "scripts/audit_tier2_leaks.py", "--json", *args],
|
||||
cwd=str(cwd),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def _files_block(result: subprocess.CompletedProcess) -> list[dict]:
|
||||
"""Parse audit --json output's 'files' block."""
|
||||
return json.loads(result.stdout)["files"]
|
||||
|
||||
|
||||
def test_audit_clean_working_tree_returns_zero(repo_root: Path) -> None:
|
||||
"""No forbidden files in working tree: audit exits 0 (informational mode)."""
|
||||
result = _run_audit(repo_root)
|
||||
assert result.returncode == 0, f"unexpected failure: {result.stderr}"
|
||||
files = _files_block(result)
|
||||
assert files == [], f"unexpected files: {files}"
|
||||
|
||||
|
||||
def test_audit_detects_forbidden_agent_file_in_working_tree(repo_root: Path) -> None:
|
||||
"""An untracked .opencode/agents/tier2-*.md file is reported."""
|
||||
opencode_dir = repo_root / ".opencode" / "agents"
|
||||
opencode_dir.mkdir(parents=True)
|
||||
(opencode_dir / "tier2-autonomous.md").write_text("leak\n")
|
||||
result = _run_audit(repo_root)
|
||||
files = _files_block(result)
|
||||
paths = {f["path"] for f in files}
|
||||
assert ".opencode/agents/tier2-autonomous.md" in paths
|
||||
|
||||
|
||||
def test_audit_detects_forbidden_command_file_in_working_tree(repo_root: Path) -> None:
|
||||
"""An untracked .opencode/commands/tier-2-*.md file is reported."""
|
||||
cmd_dir = repo_root / ".opencode" / "commands"
|
||||
cmd_dir.mkdir(parents=True)
|
||||
(cmd_dir / "tier-2-auto-execute.md").write_text("leak\n")
|
||||
result = _run_audit(repo_root)
|
||||
paths = {f["path"] for f in _files_block(result)}
|
||||
assert ".opencode/commands/tier-2-auto-execute.md" in paths
|
||||
|
||||
|
||||
def test_audit_detects_modified_opencode_json(repo_root: Path) -> None:
|
||||
"""A modified opencode.json (added to the working tree) is reported."""
|
||||
(repo_root / "opencode.json").write_text('{"tier2-modified": true}\n')
|
||||
result = _run_audit(repo_root)
|
||||
paths = {f["path"] for f in _files_block(result)}
|
||||
assert "opencode.json" in paths
|
||||
|
||||
|
||||
def test_audit_detects_modified_mcp_paths_toml(repo_root: Path) -> None:
|
||||
"""A modified mcp_paths.toml is reported."""
|
||||
(repo_root / "mcp_paths.toml").write_text('[allowed_paths]\nextra_dirs = ["leaked"]\n')
|
||||
result = _run_audit(repo_root)
|
||||
paths = {f["path"] for f in _files_block(result)}
|
||||
assert "mcp_paths.toml" in paths
|
||||
|
||||
|
||||
def test_audit_ignores_non_forbidden_files(repo_root: Path) -> None:
|
||||
"""Files NOT matching any pattern are not reported."""
|
||||
(repo_root / "src.py").write_text("print('hi')\n")
|
||||
(repo_root / "README.md").write_text("# Hello\n")
|
||||
# conductor/tier2/agents/tier2-tech-lead.md is the INTERACTIVE tier-2
|
||||
# tech-lead (main repo agent prompt), not the sandbox tier-2-autonomous.
|
||||
# It must NOT be flagged even though its path contains 'tier2-'.
|
||||
(repo_root / "conductor" / "tier2" / "agents").mkdir(parents=True, exist_ok=True)
|
||||
(repo_root / "conductor" / "tier2" / "agents" / "tier2-tech-lead.md").write_text(
|
||||
"# interactive tier-2 (allowed)\n"
|
||||
)
|
||||
result = _run_audit(repo_root)
|
||||
assert result.returncode == 0
|
||||
files = _files_block(result)
|
||||
assert files == [], f"false positives: {files}"
|
||||
|
||||
|
||||
def test_audit_reports_untracked_and_modified_separately(repo_root: Path) -> None:
|
||||
"""Untracked forbidden files: status='untracked'. Modified tracked: 'modified'."""
|
||||
# untracked case
|
||||
(repo_root / ".opencode" / "agents").mkdir(parents=True)
|
||||
(repo_root / ".opencode" / "agents" / "tier2-autonomous.md").write_text("a\n")
|
||||
result = _run_audit(repo_root)
|
||||
files = _files_block(result)
|
||||
status_by_path = {f["path"]: f["status"] for f in files}
|
||||
assert status_by_path[".opencode/agents/tier2-autonomous.md"] == "untracked"
|
||||
|
||||
|
||||
def test_audit_strict_exits_nonzero_on_leak(repo_root: Path) -> None:
|
||||
"""--strict mode: any leak causes exit 1 (CI gate)."""
|
||||
(repo_root / ".opencode" / "agents").mkdir(parents=True)
|
||||
(repo_root / ".opencode" / "agents" / "tier2-autonomous.md").write_text("leak\n")
|
||||
result = _run_audit(repo_root, "--strict")
|
||||
assert result.returncode == 1, f"strict mode should fail: {result.returncode}"
|
||||
|
||||
|
||||
def test_audit_strict_exits_zero_when_clean(repo_root: Path) -> None:
|
||||
"""--strict mode with clean tree: exit 0."""
|
||||
result = _run_audit(repo_root, "--strict")
|
||||
assert result.returncode == 0, f"strict mode should pass: {result.returncode}"
|
||||
|
||||
|
||||
def test_audit_default_mode_exits_zero_even_with_leaks(repo_root: Path) -> None:
|
||||
"""Default (informational) mode: leaks are reported but exit 0."""
|
||||
(repo_root / "opencode.json").write_text('{"leaked": true}\n')
|
||||
result = _run_audit(repo_root)
|
||||
assert result.returncode == 0, f"informational mode should pass: {result.returncode}"
|
||||
# But the leak IS reported in --json output
|
||||
files = _files_block(result)
|
||||
paths = {f["path"] for f in files}
|
||||
assert "opencode.json" in paths
|
||||
|
||||
|
||||
def test_audit_handles_missing_config_gracefully(repo_root: Path) -> None:
|
||||
"""If the forbidden-files.txt config is missing, the audit exits 0 with a
|
||||
warning. The audit should not crash on missing config (the hook would
|
||||
also no-op in this case; both layers degrade safely)."""
|
||||
(repo_root / "conductor" / "tier2" / "githooks" / "forbidden-files.txt").unlink()
|
||||
result = _run_audit(repo_root)
|
||||
assert result.returncode == 0, f"missing config should not fail: {result.stderr}"
|
||||
# No files should be reported (nothing to match against)
|
||||
assert _files_block(result) == []
|
||||
|
||||
|
||||
def test_audit_human_readable_output_includes_path(repo_root: Path) -> None:
|
||||
"""Without --json, the human-readable report mentions the leaked path."""
|
||||
(repo_root / ".opencode" / "agents").mkdir(parents=True)
|
||||
(repo_root / ".opencode" / "agents" / "tier2-autonomous.md").write_text("leak\n")
|
||||
result = subprocess.run(
|
||||
[sys.executable, "scripts/audit_tier2_leaks.py"],
|
||||
cwd=str(repo_root),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0
|
||||
assert "tier2-autonomous.md" in result.stdout, (
|
||||
f"expected path in stdout, got: {result.stdout!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_audit_summary_counts(repo_root: Path) -> None:
|
||||
"""JSON output includes a 'summary' block with total counts."""
|
||||
(repo_root / "opencode.json").write_text("a\n")
|
||||
(repo_root / "mcp_paths.toml").write_text("b\n")
|
||||
result = _run_audit(repo_root)
|
||||
data = json.loads(result.stdout)
|
||||
assert "summary" in data
|
||||
assert data["summary"]["total"] >= 2
|
||||
Reference in New Issue
Block a user