ed
eb9b8aad2e
The audit script's visit_Try had a bug where the \ or child in handler.body\ loop was OUTSIDE the \ or handler in node.handlers\ loop. So \handler\ was bound to the LAST handler, and only the last handler's body was walked. Raises in non-last except handlers were missed (e.g., src/rag_engine.py:31 was not in the audit findings). The fix moves the inner loop inside the outer loop so each handler's body is walked. Both the FIRST and LAST handler raises are now detected. Adds tests/test_audit_exception_handling_bug_fixes.py with 2 tests for the walker behavior (first-handler raise, middle-handler raise in a 3-handler try).
228 lines
8.1 KiB
Python
228 lines
8.1 KiB
Python
"""Tests for the 3 audit-script bugs documented in
|
|
docs/reports/RESULT_MIGRATION_REVIEW_PASS_20260617.md section 4.4.
|
|
|
|
Bug 1: visit_Try walker only walks children of the LAST except handler.
|
|
Raises in non-last except handlers are missed.
|
|
Bug 2: render_json filters out compliant findings in non-verbose mode.
|
|
INTERNAL_COMPLIANT sites are not in the per-file findings list.
|
|
Bug 3: render_json truncates per-file list to --top (default 15).
|
|
Low-violation files with UNCLEAR sites are not in the per-file list.
|
|
|
|
Each test uses the subprocess pattern from test_audit_exception_handling_heuristics.py:
|
|
create a fixture file, run the audit, parse the JSON, assert.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SCRIPT = ROOT / "scripts" / "audit_exception_handling.py"
|
|
|
|
|
|
def _run_audit_on_fixture(source: str, top: int = 15, verbose: bool = False) -> dict:
|
|
"""Run the audit script on a fixture and return the parsed JSON output."""
|
|
import tempfile
|
|
tmpdir = Path(tempfile.mkdtemp(prefix="audit_bugfix_fixture_"))
|
|
fixture = tmpdir / "audit_bugfix_fixture.py"
|
|
fixture.write_text(textwrap.dedent(source), encoding="utf-8")
|
|
try:
|
|
cmd = [sys.executable, str(SCRIPT), "--json", "--src", str(tmpdir),
|
|
"--top", str(top), "--include-baseline"]
|
|
if verbose:
|
|
cmd.append("--verbose")
|
|
result = subprocess.run(
|
|
cmd,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
cwd=str(ROOT),
|
|
)
|
|
finally:
|
|
if fixture.exists():
|
|
fixture.unlink()
|
|
if tmpdir.exists():
|
|
tmpdir.rmdir()
|
|
if result.returncode not in (0, 1):
|
|
raise RuntimeError(f"audit failed: {result.stderr}")
|
|
return json.loads(result.stdout)
|
|
|
|
|
|
def _classifications_for_file(data: dict, filename_suffix: str) -> list[dict]:
|
|
"""Return all findings for files whose path ends with `filename_suffix`."""
|
|
return [
|
|
f
|
|
for file_info in data.get("files", [])
|
|
for f in file_info.get("findings", [])
|
|
if file_info["filename"].endswith(filename_suffix)
|
|
]
|
|
|
|
|
|
def _has_raise_finding(findings: list[dict], target_line: int) -> bool:
|
|
"""True if there is a RAISE-kind finding at the target line."""
|
|
return any(
|
|
f.get("kind") == "RAISE" and f.get("line") == target_line
|
|
for f in findings
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Bug 1: visit_Try walker only walks children of the LAST except handler
|
|
# ---------------------------------------------------------------------------
|
|
def test_visit_try_walks_first_handler_raise():
|
|
"""A raise in the FIRST except handler must be detected.
|
|
|
|
The audit script's visit_Try has a bug: the `for child in handler.body`
|
|
loop is OUTSIDE the `for handler in node.handlers` loop. So `handler` is
|
|
bound to the LAST handler, and only the last handler's body is walked.
|
|
|
|
Per docs/reports/RESULT_MIGRATION_REVIEW_PASS_20260617.md section 4.4 #1:
|
|
src/rag_engine.py:31 (raise in the FIRST except of a 2-handler try)
|
|
is missed by the audit. After the fix, both raises should be reported.
|
|
"""
|
|
src = '''
|
|
def func():
|
|
try:
|
|
from foo import bar
|
|
return bar()
|
|
except ImportError as e:
|
|
if e.name == "foo":
|
|
raise RuntimeError("missing foo") from e
|
|
raise
|
|
except Exception as e:
|
|
sys.stderr.write(f"other error: {e}")
|
|
raise e
|
|
'''
|
|
data = _run_audit_on_fixture(src, verbose=True)
|
|
findings = _classifications_for_file(data, "audit_bugfix_fixture.py")
|
|
raise_lines = sorted(f["line"] for f in findings if f.get("kind") == "RAISE")
|
|
# The first except's raise is at line 8 (raise RuntimeError).
|
|
# The second except's raise is at line 12 (raise e).
|
|
# After the fix, BOTH lines should be in the findings.
|
|
# Before the fix, only line 12 (last handler) is reported.
|
|
# Use --verbose to bypass the Bug 2 (render_json filter) which hides
|
|
# INTERNAL_PROGRAMMER_RAISE findings in non-verbose mode.
|
|
assert 8 in raise_lines, (
|
|
f"visit_Try walker bug: first except handler's raise at line 8 "
|
|
f"is not in findings. Got raise lines: {raise_lines}"
|
|
)
|
|
|
|
|
|
def test_visit_try_walks_middle_handler_raise():
|
|
"""A raise in the MIDDLE except handler must be detected (3-handler try)."""
|
|
src = '''
|
|
def func():
|
|
try:
|
|
from foo import bar
|
|
return bar()
|
|
except ImportError as e:
|
|
pass
|
|
except ValueError as e:
|
|
raise ValueError("rewrapped") from e
|
|
except Exception as e:
|
|
raise e
|
|
'''
|
|
data = _run_audit_on_fixture(src, verbose=True)
|
|
findings = _classifications_for_file(data, "audit_bugfix_fixture.py")
|
|
raise_lines = sorted(f["line"] for f in findings if f.get("kind") == "RAISE")
|
|
# middle handler's `raise ValueError(...)` is at line 9 in the dedented source.
|
|
# last handler's `raise e` is at line 11.
|
|
# Both must be in the findings after the visit_Try fix.
|
|
assert 9 in raise_lines, (
|
|
f"visit_Try walker bug: middle except handler's raise at line 9 "
|
|
f"is not in findings. Got raise lines: {raise_lines}"
|
|
)
|
|
assert 11 in raise_lines, (
|
|
f"last handler's raise at line 11 must also be detected. "
|
|
f"Got raise lines: {raise_lines}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Bug 2: render_json filters out INTERNAL_COMPLIANT findings in non-verbose mode
|
|
# ---------------------------------------------------------------------------
|
|
def test_render_json_includes_compliant_findings():
|
|
"""INTERNAL_COMPLIANT findings must appear in the per-file list.
|
|
|
|
Per docs/reports/RESULT_MIGRATION_REVIEW_PASS_20260617.md section 4.4 #2:
|
|
The render_json filter `if f.category in VIOLATION_CATEGORIES or
|
|
f.category in ("UNCLEAR", "INTERNAL_RETHROW")` excludes
|
|
INTERNAL_COMPLIANT findings from the per-file list in non-verbose mode.
|
|
|
|
The fix: include all findings in the per-file list (totals are right;
|
|
per-file list should match).
|
|
"""
|
|
src = '''
|
|
def func(items, target):
|
|
try:
|
|
idx = items.index(target)
|
|
except ValueError:
|
|
idx = 0
|
|
return idx
|
|
'''
|
|
data = _run_audit_on_fixture(src)
|
|
findings = _classifications_for_file(data, "audit_bugfix_fixture.py")
|
|
categories = [f.get("category") for f in findings]
|
|
# The list.index/ValueError fallback is INTERNAL_COMPLIANT per the
|
|
# heuristic added in the review pass. It must appear in the per-file list.
|
|
assert "INTERNAL_COMPLIANT" in categories, (
|
|
f"INTERNAL_COMPLIANT finding is filtered out of the per-file list. "
|
|
f"Got categories: {categories}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Bug 3: render_json truncates per-file list to --top (default 15)
|
|
# ---------------------------------------------------------------------------
|
|
def test_render_json_no_truncation_with_20_files():
|
|
"""The per-file list is NOT truncated to top 15 by default.
|
|
|
|
Per docs/reports/RESULT_MIGRATION_REVIEW_PASS_20260617.md section 4.4 #3:
|
|
The per-file list is truncated to top 15 by default. With 20 files,
|
|
the 5 lowest-ranked files are excluded from the output.
|
|
|
|
The fix: increase the default --top to >= total file count.
|
|
"""
|
|
# Create 20 files each with 1 violation. Run audit with default --top.
|
|
# All 20 files should appear in the per-file list.
|
|
# With the bug (top=15), only 15 of the 20 files appear.
|
|
import tempfile
|
|
tmpdir = Path(tempfile.mkdtemp(prefix="audit_bugfix_truncation_"))
|
|
try:
|
|
for i in range(20):
|
|
src = f'''
|
|
def func_{i:02d}():
|
|
try:
|
|
return some_undefined()
|
|
except Exception as e:
|
|
logger.error("{{e}}")
|
|
return None
|
|
'''
|
|
(tmpdir / f"truncation_{i:02d}.py").write_text(textwrap.dedent(src), encoding="utf-8")
|
|
result = subprocess.run(
|
|
[sys.executable, str(SCRIPT), "--json", "--src", str(tmpdir),
|
|
"--include-baseline"],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
cwd=str(ROOT),
|
|
)
|
|
if result.returncode not in (0, 1):
|
|
raise RuntimeError(f"audit failed: {result.stderr}")
|
|
data = json.loads(result.stdout)
|
|
filenames = [f["filename"] for f in data.get("files", [])]
|
|
truncation_files = [fn for fn in filenames if "truncation_" in fn]
|
|
finally:
|
|
for f in tmpdir.glob("*.py"):
|
|
f.unlink()
|
|
if tmpdir.exists():
|
|
tmpdir.rmdir()
|
|
# All 20 files should be in the output. With the bug, only 15 appear.
|
|
assert len(truncation_files) == 20, (
|
|
f"per-file list is truncated to top 15. Got {len(truncation_files)}/20 "
|
|
f"files: {truncation_files}"
|
|
)
|