ed
c5dbfd6edf
3 regression tests for the new Heuristic E (narrow + structured error carrier):
1. test_heuristic_e_narrow_return_errorinfo_is_compliant
- Asserts narrow except + return ErrorInfo(...) is classified as compliant
- Accepts both INTERNAL_COMPLIANT (Heuristic E) and BOUNDARY_CONVERSION
(existing creates_errorinfo check, fires first)
2. test_heuristic_e_narrow_dict_error_true_assign_is_compliant
- Asserts narrow except + dict[error] = True is classified as compliant
- The in-band error flag pattern (per Tier 1 directive)
3. test_heuristic_e_empty_default_args_is_NOT_compliant
- NEGATIVE test: narrow except + args = {} must NOT be classified as compliant
- Guards against future heuristic additions that would laundering the
sliming empty-default pattern (per TIER1_REVIEW)
Total: 16 audit heuristic tests pass (13 existing + 3 new).
389 lines
16 KiB
Python
389 lines
16 KiB
Python
# Phase 7 Task 7.8 - Regression-guard tests for audit heuristic.
|
|
# Per Phase 7 spec 22.5.5 (FR5):
|
|
# - BOUNDARY_FASTAPI classification requires ast.Raise(exc=HTTPException)
|
|
# OR a return of Result(...) in the except body.
|
|
# - Otherwise re-classify as INTERNAL_SILENT_SWALLOW (logging body) or
|
|
# INTERNAL_COMPLIANT (try/finally cleanup).
|
|
import ast
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
|
|
from audit_exception_handling import ( # noqa: E402
|
|
ExceptionVisitor,
|
|
audit_file,
|
|
)
|
|
|
|
|
|
def _make_visitor(source: str, func_name: str):
|
|
"""Create an ExceptionVisitor positioned inside the named function."""
|
|
tree = ast.parse(source)
|
|
visitor = ExceptionVisitor(str(ROOT / "src" / "_test_dummy.py"))
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.FunctionDef) and node.name == func_name:
|
|
visitor._func_stack = [node]
|
|
return visitor
|
|
raise ValueError(f"Function {func_name} not found in source")
|
|
|
|
|
|
def _find_handler(visitor):
|
|
"""Find the first Try node in the function body."""
|
|
for node in visitor._func_stack[0].body:
|
|
if isinstance(node, ast.Try):
|
|
return node
|
|
raise AssertionError("expected a try/except in function")
|
|
|
|
|
|
def test_is_api_handler_requires_http_exception_in_body():
|
|
# OLD STYLE: only stderr.write (should NOT be BOUNDARY_FASTAPI after Phase 7)
|
|
src = (
|
|
"def _api_generate(controller):\n"
|
|
" HTTPException = None\n"
|
|
" try:\n"
|
|
" do_something()\n"
|
|
" except Exception as e:\n"
|
|
" sys.stderr.write('err: ' + str(e))\n"
|
|
" sys.stderr.flush()\n"
|
|
)
|
|
visitor = _make_visitor(src, "_api_generate")
|
|
handler = _find_handler(visitor).handlers[0]
|
|
category, _ = visitor._classify_except(handler, _find_handler(visitor))
|
|
assert category != "BOUNDARY_FASTAPI", (
|
|
f"Phase 7 FR5 tightening failed: stale body (only stderr.write) "
|
|
f"should NOT be BOUNDARY_FASTAPI; got {category}."
|
|
)
|
|
|
|
|
|
def test_api_handler_with_http_exception_raise_is_boundary_fastapi():
|
|
# NEW STYLE: raises HTTPException (the canonical FastAPI pattern)
|
|
src = (
|
|
"def _api_generate(controller):\n"
|
|
" HTTPException = None\n"
|
|
" try:\n"
|
|
" do_something()\n"
|
|
" except Exception as e:\n"
|
|
" raise HTTPException(status_code=500, detail=str(e))\n"
|
|
)
|
|
visitor = _make_visitor(src, "_api_generate")
|
|
try_node = _find_handler(visitor)
|
|
handler = try_node.handlers[0]
|
|
category, _ = visitor._classify_except(handler, try_node)
|
|
assert category == "BOUNDARY_FASTAPI", (
|
|
f"Phase 7 FR5 regression: handler with HTTPException raise should be "
|
|
f"BOUNDARY_FASTAPI; got {category}."
|
|
)
|
|
|
|
|
|
def test_non_api_handler_with_logging_is_still_internal_compliant():
|
|
# Non-_api_* function with logging-only except body
|
|
src = (
|
|
"def regular_handler():\n"
|
|
" try:\n"
|
|
" do_something()\n"
|
|
" except Exception as e:\n"
|
|
" logging.getLogger('x').debug('err: %s', e)\n"
|
|
" print('err: ' + str(e))\n"
|
|
)
|
|
visitor = _make_visitor(src, "regular_handler")
|
|
try_node = _find_handler(visitor)
|
|
handler = try_node.handlers[0]
|
|
category, _ = visitor._classify_except(handler, try_node)
|
|
assert category in ("INTERNAL_COMPLIANT", "INTERNAL_SILENT_SWALLOW", "INTERNAL_BROAD_CATCH"), (
|
|
f"Non-api handler should NOT be BOUNDARY_FASTAPI; got {category}."
|
|
)
|
|
|
|
|
|
def test_15_existing_fastapi_sites_remain_classified():
|
|
# The 13 BOUNDARY_FASTAPI sites in src/app_controller.py must remain
|
|
# classified after the heuristic tightening (Phase 7 FR5).
|
|
# Note: src/api_hooks.py functions do NOT have _api_ prefix, so they
|
|
# were never classified BOUNDARY_FASTAPI; the 13 sites are all in
|
|
# _api_* handlers in app_controller.py.
|
|
app_controller_path = ROOT / "src" / "app_controller.py"
|
|
if not app_controller_path.exists():
|
|
pytest.skip(f"{app_controller_path} not found")
|
|
report = audit_file(app_controller_path)
|
|
fastapi_sites = [f for f in report.findings if f.category == "BOUNDARY_FASTAPI"]
|
|
assert len(fastapi_sites) >= 10, (
|
|
f"Phase 7 regression: expected at least 10 BOUNDARY_FASTAPI sites in "
|
|
f"src/app_controller.py, got {len(fastapi_sites)}. The known sites "
|
|
f"must remain classified after heuristic tightening."
|
|
)
|
|
src = app_controller_path.read_text(encoding="utf-8")
|
|
for site in fastapi_sites[:3]:
|
|
lines = src.split("\n")
|
|
line_num = site.line
|
|
window = "\n".join(lines[max(0, line_num - 5):line_num + 5])
|
|
assert "HTTPException" in window or "Result[" in window, (
|
|
f"Phase 7 regression: site at app_controller.py:{line_num} "
|
|
f"classified BOUNDARY_FASTAPI but window doesn't contain "
|
|
f"HTTPException or Result["
|
|
)
|
|
|
|
|
|
def test_phase7_migrated_sites_no_longer_silent_swallow():
|
|
# L242/L256/L5064/L5093 must not be INTERNAL_SILENT_SWALLOW after Phase 7.
|
|
app_controller_path = ROOT / "src" / "app_controller.py"
|
|
if not app_controller_path.exists():
|
|
pytest.skip(f"{app_controller_path} not found")
|
|
report = audit_file(app_controller_path)
|
|
for f in report.findings:
|
|
if f.line in (242, 256, 5064, 5093):
|
|
assert f.category != "INTERNAL_SILENT_SWALLOW", (
|
|
f"Phase 7 regression: L{f.line} should not be "
|
|
f"INTERNAL_SILENT_SWALLOW after migration; got {f.category}"
|
|
)
|
|
|
|
|
|
# Phase 11 Task 11.4 - Regression-guard tests for dunder-method bare-raise heuristic.
|
|
# Per Phase 11 spec (INTERNAL_RETHROW classification for dunder methods):
|
|
# - Bare `raise AttributeError(name)` / `raise NameError(name)` in
|
|
# `__getattr__`, `__getattribute__`, `__setattr__`, `__delattr__` is the
|
|
# canonical dunder-method programmer-error pattern (per styleguide
|
|
# "Re-Raise Patterns": bare raises are reserved for programmer errors).
|
|
# - The audit previously classified these as INTERNAL_RETHROW (suspicious);
|
|
# the heuristic must reclassify them as INTERNAL_PROGRAMMER_RAISE.
|
|
|
|
DUNDER_RAISE_TESTS = {
|
|
"__getattr__": "def __getattr__(self, name):\n if name == 'controller':\n raise AttributeError(name)\n raise AttributeError(name)",
|
|
"__getattribute__": "def __getattribute__(self, name):\n if name == 'controller':\n raise AttributeError(name)\n raise AttributeError(name)",
|
|
"__setattr__": "def __setattr__(self, name, value):\n raise AttributeError(name)",
|
|
"__delattr__": "def __delattr__(self, name):\n raise AttributeError(name)",
|
|
}
|
|
|
|
|
|
def _classify_first_raise(source, func_name):
|
|
tree = ast.parse(source)
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.FunctionDef) and node.name == func_name:
|
|
visitor = ExceptionVisitor(str(ROOT / "src" / "_test_dummy.py"))
|
|
visitor._func_stack = [node]
|
|
for sub in ast.walk(node):
|
|
if isinstance(sub, ast.Raise) and sub.exc is not None:
|
|
return visitor._classify_raise(sub)
|
|
raise AssertionError(f"No raise found in {func_name}")
|
|
|
|
|
|
def test_bare_raise_attribute_error_in_getattr_is_programmer_raise():
|
|
src = DUNDER_RAISE_TESTS["__getattr__"]
|
|
category, hint = _classify_first_raise(src, "__getattr__")
|
|
assert category == "INTERNAL_PROGRAMMER_RAISE", (
|
|
f"Phase 11 regression: bare `raise AttributeError(name)` in __getattr__ "
|
|
f"should be INTERNAL_PROGRAMMER_RAISE (canonical dunder-method pattern); "
|
|
f"got {category}. Hint: {hint}"
|
|
)
|
|
|
|
|
|
def test_bare_raise_name_error_in_getattr_is_programmer_raise():
|
|
src = (
|
|
"def __getattr__(self, name):\n"
|
|
" if not hasattr(self, 'x'):\n"
|
|
" raise NameError(name)\n"
|
|
" return self.x"
|
|
)
|
|
category, hint = _classify_first_raise(src, "__getattr__")
|
|
assert category == "INTERNAL_PROGRAMMER_RAISE", (
|
|
f"Phase 11 regression: bare `raise NameError(name)` in __getattr__ "
|
|
f"should be INTERNAL_PROGRAMMER_RAISE (canonical dunder-method pattern); "
|
|
f"got {category}. Hint: {hint}"
|
|
)
|
|
|
|
|
|
def test_bare_raise_in_setattr_is_programmer_raise():
|
|
src = DUNDER_RAISE_TESTS["__setattr__"]
|
|
category, hint = _classify_first_raise(src, "__setattr__")
|
|
assert category == "INTERNAL_PROGRAMMER_RAISE", (
|
|
f"Phase 11 regression: bare `raise AttributeError` in __setattr__ "
|
|
f"should be INTERNAL_PROGRAMMER_RAISE (canonical dunder-method pattern); "
|
|
f"got {category}. Hint: {hint}"
|
|
)
|
|
|
|
|
|
def test_bare_raise_in_delattr_is_programmer_raise():
|
|
src = DUNDER_RAISE_TESTS["__delattr__"]
|
|
category, hint = _classify_first_raise(src, "__delattr__")
|
|
assert category == "INTERNAL_PROGRAMMER_RAISE", (
|
|
f"Phase 11 regression: bare `raise AttributeError` in __delattr__ "
|
|
f"should be INTERNAL_PROGRAMMER_RAISE (canonical dunder-method pattern); "
|
|
f"got {category}. Hint: {hint}"
|
|
)
|
|
|
|
|
|
def test_bare_raise_in_getattribute_is_programmer_raise():
|
|
src = DUNDER_RAISE_TESTS["__getattribute__"]
|
|
category, hint = _classify_first_raise(src, "__getattribute__")
|
|
assert category == "INTERNAL_PROGRAMMER_RAISE", (
|
|
f"Phase 11 regression: bare `raise AttributeError(name)` in __getattribute__ "
|
|
f"should be INTERNAL_PROGRAMMER_RAISE (canonical dunder-method pattern); "
|
|
f"got {category}. Hint: {hint}"
|
|
)
|
|
|
|
|
|
# Phase 12 Task 12.1 - Regression-guard tests for the lazy-loading sentinel
|
|
# fallback heuristic.
|
|
# Per Phase 12 spec (INTERNAL_COMPLIANT classification for lazy-loading
|
|
# sentinel fallbacks in methods named _resolve/_load/_get/_try_load):
|
|
# - The except body must NOT re-raise
|
|
# - The except body must assign to a self.<attr> (directly or via nested try)
|
|
# - The except set must be in {AttributeError, ImportError, ModuleNotFoundError}
|
|
# - The enclosing function name must be in the lazy-loader set
|
|
# Pre-Phase 12 baseline: 2 UNCLEAR sites in src/gui_2.py at L65, L69
|
|
# (both in _LazyModule._resolve). Post-Phase 12: 0 UNCLEAR.
|
|
|
|
def test_lazy_loading_sentinel_fallback_in_resolve_is_compliant():
|
|
src = (
|
|
"def _resolve(self):\n"
|
|
" try:\n"
|
|
" self._cached = getattr(self._mod, self._attr_name)\n"
|
|
" except AttributeError:\n"
|
|
" try:\n"
|
|
" self._cached = _importlib.import_module(self._sub_name)\n"
|
|
" except (ImportError, ModuleNotFoundError):\n"
|
|
" self._cached = _FiledialogStub()\n"
|
|
" return self._cached\n"
|
|
)
|
|
visitor = _make_visitor(src, "_resolve")
|
|
try_node = _find_handler(visitor)
|
|
handler = try_node.handlers[0]
|
|
category, hint = visitor._classify_except(handler, try_node)
|
|
assert category == "INTERNAL_COMPLIANT", (
|
|
f"Phase 12 regression: lazy-loading sentinel fallback in `_resolve` "
|
|
f"(L65-style nested try with `self._cached = _FiledialogStub()`) "
|
|
f"should be INTERNAL_COMPLIANT (canonical graceful-degradation pattern); "
|
|
f"got {category}. Hint: {hint}"
|
|
)
|
|
|
|
|
|
def test_lazy_loading_sentinel_fallback_in_load_is_compliant():
|
|
src = (
|
|
"def _load(self, name):\n"
|
|
" try:\n"
|
|
" self._cached = _importlib.import_module(name)\n"
|
|
" except (ImportError, ModuleNotFoundError):\n"
|
|
" self._cached = _FooStub()\n"
|
|
" return self._cached\n"
|
|
)
|
|
visitor = _make_visitor(src, "_load")
|
|
try_node = _find_handler(visitor)
|
|
handler = try_node.handlers[0]
|
|
category, hint = visitor._classify_except(handler, try_node)
|
|
assert category == "INTERNAL_COMPLIANT", (
|
|
f"Phase 12 regression: lazy-loading sentinel fallback in `_load` "
|
|
f"(direct `self._cached = _FooStub()`) should be INTERNAL_COMPLIANT "
|
|
f"(canonical graceful-degradation pattern); got {category}. Hint: {hint}"
|
|
)
|
|
|
|
|
|
def test_lazy_loading_sentinel_fallback_in_get_is_compliant():
|
|
src = (
|
|
"def _get(self, attr_name):\n"
|
|
" try:\n"
|
|
" return getattr(self._module, attr_name)\n"
|
|
" except AttributeError:\n"
|
|
" self._cached = _BarStub()\n"
|
|
" return self._cached\n"
|
|
)
|
|
visitor = _make_visitor(src, "_get")
|
|
try_node = _find_handler(visitor)
|
|
handler = try_node.handlers[0]
|
|
category, hint = visitor._classify_except(handler, try_node)
|
|
assert category == "INTERNAL_COMPLIANT", (
|
|
f"Phase 12 regression: lazy-loading sentinel fallback in `_get` "
|
|
f"(direct `self._cached = _BarStub()`) should be INTERNAL_COMPLIANT "
|
|
f"(canonical graceful-degradation pattern); got {category}. Hint: {hint}"
|
|
)
|
|
|
|
|
|
# ============ Phase 9 redo: Heuristic E regression tests (TIER1_REVIEW) ============
|
|
|
|
def test_heuristic_e_narrow_return_errorinfo_is_compliant():
|
|
"""Phase 9 redo: narrow except + return ErrorInfo(...) is a true drain.
|
|
|
|
Per TIER1_REVIEW_phase9_dilemma_20260620: a narrow except body that
|
|
returns a structured ErrorInfo carries the original exception and is
|
|
the function's contract. This is NOT sliming (the error context is
|
|
preserved in `original=e`).
|
|
"""
|
|
src = (
|
|
"def _classify_anthropic_error(exc, source):\n"
|
|
" try:\n"
|
|
" err_data = exc.response.json()\n"
|
|
" except (ValueError, AttributeError) as e:\n"
|
|
" return ErrorInfo(kind=ErrorKind.UNKNOWN, message=str(e), source=source, original=e)\n"
|
|
)
|
|
visitor = _make_visitor(src, "_classify_anthropic_error")
|
|
try_node = _find_handler(visitor)
|
|
handler = try_node.handlers[0]
|
|
category, hint = visitor._classify_except(handler, try_node)
|
|
assert category in ("INTERNAL_COMPLIANT", "BOUNDARY_CONVERSION"), (
|
|
f"Heuristic E regression: narrow except + return ErrorInfo(...) "
|
|
f"should be a compliant classification (INTERNAL_COMPLIANT via Heuristic E "
|
|
f"or BOUNDARY_CONVERSION via existing creates_errorinfo check); got {category}. Hint: {hint}"
|
|
)
|
|
|
|
|
|
def test_heuristic_e_narrow_dict_error_true_assign_is_compliant():
|
|
"""Phase 9 redo: narrow except + dict[error] = True is a true drain (in-band flag).
|
|
|
|
Per TIER1_REVIEW: `except (NarrowType) as e: item["error"] = True`
|
|
is a structured error carrier. The caller is expected to inspect the
|
|
`error` flag (per-site decision documented in track notes; the audit
|
|
does NOT verify caller reads the flag).
|
|
"""
|
|
src = (
|
|
"def _reread_file_items(file_items):\n"
|
|
" try:\n"
|
|
" content = p.read_text()\n"
|
|
" new_item = {**item, 'content': content}\n"
|
|
" except (OSError, UnicodeDecodeError) as e:\n"
|
|
" err_item = {**item, 'content': f'ERROR: {e}'}\n"
|
|
" err_item['error'] = True\n"
|
|
" refreshed.append(err_item)\n"
|
|
)
|
|
visitor = _make_visitor(src, "_reread_file_items")
|
|
try_node = _find_handler(visitor)
|
|
handler = try_node.handlers[0]
|
|
category, hint = visitor._classify_except(handler, try_node)
|
|
assert category == "INTERNAL_COMPLIANT", (
|
|
f"Heuristic E regression: narrow except + dict['error'] = True "
|
|
f"should be INTERNAL_COMPLIANT (in-band error flag carrier); got {category}. Hint: {hint}"
|
|
)
|
|
|
|
|
|
def test_heuristic_e_empty_default_args_is_NOT_compliant():
|
|
"""Phase 9 redo: narrow except + args = {} is NOT a drain (sliming).
|
|
|
|
Per TIER1_REVIEW: the empty-default pattern loses error context. The
|
|
caller cannot distinguish success from failure. Heuristic E
|
|
explicitly does NOT match this pattern (this test is a regression
|
|
guard against future "helpful" heuristic additions that would
|
|
laundering this sliming pattern).
|
|
|
|
Structure: extract into a helper function so the try is at the top
|
|
level of the function body (required by _find_handler test helper).
|
|
"""
|
|
src = (
|
|
"def _parse_tool_args(tool_args_str):\n"
|
|
" try:\n"
|
|
" args = json.loads(tool_args_str)\n"
|
|
" except (ValueError, TypeError):\n"
|
|
" args = {}\n"
|
|
" return args\n"
|
|
)
|
|
visitor = _make_visitor(src, "_parse_tool_args")
|
|
try_node = _find_handler(visitor)
|
|
handler = try_node.handlers[0]
|
|
category, hint = visitor._classify_except(handler, try_node)
|
|
# The site is narrow + non-broad but the body is empty-default.
|
|
# Heuristic E should NOT classify as COMPLIANT. May be INTERNAL_BROAD_CATCH
|
|
# (no drain) or UNCLEAR. NOT INTERNAL_COMPLIANT or BOUNDARY_CONVERSION.
|
|
assert category not in ("INTERNAL_COMPLIANT", "BOUNDARY_CONVERSION"), (
|
|
f"Heuristic E regression: narrow except + args = {{}} (empty default) "
|
|
f"must NOT be classified as compliant (INTERNAL_COMPLIANT or BOUNDARY_CONVERSION "
|
|
f"would be sliming per TIER1_REVIEW). Got {category} which would laundering the pattern. Hint: {hint}"
|
|
)
|