manual_slop/tests/test_baseline_result.py

"""Invariant tests for result_migration_baseline_cleanup_20260620.

Phase 1 (4): audit + inventory doc counts match expected baseline
Phase 2 (3): baseline state is correct (88 MIG sites in 3 files)
Phase 3 (3): mcp_client BC count decreased from 40 -> 32 after Batch A
"""
import json
import subprocess
from collections import Counter
from pathlib import Path

import pytest


AUDIT_PATH = Path("tests/artifacts/PHASE1_AUDIT_BASELINE.json")
INV_MCP = Path("tests/artifacts/PHASE1_INVENTORY_mcp_client.md")
INV_AI = Path("tests/artifacts/PHASE1_INVENTORY_ai_client.md")
INV_RAG = Path("tests/artifacts/PHASE1_INVENTORY_rag_engine.md")

MIG = {"INTERNAL_BROAD_CATCH", "INTERNAL_SILENT_SWALLOW", "INTERNAL_OPTIONAL_RETURN", "INTERNAL_RETHROW", "UNCLEAR"}
EXPECTED = {
    "src\\mcp_client.py": (40, 5, 0, 0, 1, 46),
    "src\\ai_client.py": (17, 9, 0, 7, 0, 33),
    "src\\rag_engine.py": (5, 1, 0, 3, 0, 9),
}
TARGETS = ("src\\mcp_client.py", "src\\ai_client.py", "src\\rag_engine.py")


def _load_audit():
    return json.loads(AUDIT_PATH.read_text(encoding="utf-8"))


def _audit_live():
    r = subprocess.run(
        ["uv", "run", "python", "scripts/audit_exception_handling.py",
         "--include-baseline", "--json"],
        capture_output=True, text=True
    )
    return json.loads(r.stdout)


# ============ Phase 1 tests (4) ============

def test_phase1_audit_json_exists():
    assert AUDIT_PATH.exists(), f"missing audit json at {AUDIT_PATH}"


def test_phase1_inventory_docs_exist():
    for p in [INV_MCP, INV_AI, INV_RAG]:
        assert p.exists(), f"missing inventory doc at {p}"
        assert p.stat().st_size > 500, f"inventory doc {p} too small"


def test_phase1_total_migration_target_is_88():
    data = _load_audit()
    files = {f["filename"]: f for f in data["files"]}
    total = 0
    for key in EXPECTED:
        findings = files[key]["findings"]
        mig = [f for f in findings if f["category"] in MIG]
        total += len(mig)
    assert total == 88, f"expected 88 migration-target sites, got {total}"


def test_phase1_per_file_site_counts():
    data = _load_audit()
    files = {f["filename"]: f for f in data["files"]}
    for key, expected in EXPECTED.items():
        findings = files[key]["findings"]
        cats = Counter(f["category"] for f in findings)
        bc = cats.get("INTERNAL_BROAD_CATCH", 0)
        ss = cats.get("INTERNAL_SILENT_SWALLOW", 0)
        opt = cats.get("INTERNAL_OPTIONAL_RETURN", 0)
        rethrow = cats.get("INTERNAL_RETHROW", 0)
        unclear = cats.get("UNCLEAR", 0)
        mig = bc + ss + opt + rethrow + unclear
        assert (bc, ss, opt, rethrow, unclear, mig) == expected, (
            f"{key}: expected BC={expected[0]} SS={expected[1]} OPT={expected[2]} "
            f"RETHROW={expected[3]} UNCLEAR={expected[4]} MIG={expected[5]}, "
            f"got BC={bc} SS={ss} OPT={opt} RETHROW={rethrow} UNCLEAR={unclear} MIG={mig}"
        )


# ============ Phase 2 tests (3) ============

def test_phase2_baseline_audit_runs():
    r = subprocess.run(
        ["uv", "run", "python", "scripts/audit_exception_handling.py",
         "--include-baseline", "--json"],
        capture_output=True, text=True
    )
    assert r.returncode == 0, f"audit failed: {r.stderr[:500]}"
    data = json.loads(r.stdout)
    assert "files" in data
    assert len(data["files"]) >= 40, f"expected 40+ files, got {len(data['files'])}"


def test_phase2_all_3_targets_have_migration_sites():
    data = _load_audit()
    files = {f["filename"]: f for f in data["files"]}
    for target in TARGETS:
        assert target in files, f"missing target file: {target}"
        mig = [f for f in files[target]["findings"] if f["category"] in MIG]
        assert len(mig) > 0, f"{target} has 0 migration-target sites (expected >0)"


def test_phase2_per_file_baseline_counts_match_inventory():
    data = _load_audit()
    files = {f["filename"]: f for f in data["files"]}
    BASELINE = {"src\\mcp_client.py": 46, "src\\ai_client.py": 33, "src\\rag_engine.py": 9}
    for target, expected in BASELINE.items():
        mig = [f for f in files[target]["findings"] if f["category"] in MIG]
        assert len(mig) == expected, (
            f"{target}: baseline expected {expected}, got {len(mig)} "
            f"(a previous phase may have introduced a violation)"
        )


# ============ Phase 3 tests (3) ============

def test_phase3_mcp_client_broad_catch_decreased_from_40_to_32():
    data = _audit_live()
    files = {f["filename"]: f for f in data["files"]}
    findings = files["src\\mcp_client.py"]["findings"]
    bc = sum(1 for f in findings if f["category"] == "INTERNAL_BROAD_CATCH")
    assert bc == 32, f"expected mcp_client BC=32 after Batch A, got {bc}"


def test_phase3_total_migration_target_decreased_to_80():
    data = _audit_live()
    files = {f["filename"]: f for f in data["files"]}
    total = 0
    for key in TARGETS:
        findings = files[key]["findings"]
        total += sum(1 for f in findings if f["category"] in MIG)
    assert total == 80, f"expected total MIG=80 after Phase 3, got {total}"


def test_phase3_audit_baseline_matches_phase1_audit_json():
    data = _load_audit()
    files = {f["filename"]: f for f in data["files"]}
    total = 0
    for key in TARGETS:
        findings = files[key]["findings"]
        total += sum(1 for f in findings if f["category"] in MIG)
    assert total == 88, f"PHASE1_AUDIT_BASELINE.json expected 88 baseline MIG, got {total}"