test(stabilization): Align tier4_interceptor tests with Popen and integrate vlogger

This commit is contained in:
2026-02-28 20:20:17 -05:00
parent dfb4fa1b26
commit 5e320b2bbf
2 changed files with 66 additions and 43 deletions

View File

@@ -45,10 +45,11 @@
- **Requirement:** Implement "Before vs After" state tables in `logs/test/unique_signature/<script_name>.txt`. - **Requirement:** Implement "Before vs After" state tables in `logs/test/unique_signature/<script_name>.txt`.
- [x] Task: Conductor - Fix `ai_style_formatter.py` test expectations for ultra-compact style. - [x] Task: Conductor - Fix `ai_style_formatter.py` test expectations for ultra-compact style.
- **Requirement:** Log table-based comparison of code states in `logs/test/unique_signature/test_ai_style_formatter.txt`. - **Requirement:** Log table-based comparison of code states in `logs/test/unique_signature/test_ai_style_formatter.txt`.
- [ ] Task: Conductor - Align `tier4_interceptor.py` tests with current PowerShell output formatting. - [x] Task: Conductor - Align `tier4_interceptor.py` tests with current PowerShell output formatting.
- **Requirement:** Log expected vs actual PowerShell output mappings in `logs/test/unique_signature/test_tier4_interceptor.txt`. - **Requirement:** Log expected vs actual PowerShell output mappings in `logs/test/unique_signature/test_tier4_interceptor.txt`.
- [ ] Task: Conductor - Investigate and stabilize `live_gui` test environment collection/startup. - [ ] Task: Conductor - Investigate and stabilize `live_gui` test environment collection/startup.
- **Requirement:** Log comprehensive environment telemetry (Before/After/Delta) in `logs/test/unique_signature/live_gui_diag.txt`. - **Requirement:** Log comprehensive environment telemetry (Before/After/Delta) in `logs/test/unique_signature/live_gui_diag.txt`.
- [ ] Task: Conductor - User Manual Verification 'Phase 6: Test Suite Stabilization' - [ ] Task: Conductor - User Manual Verification 'Phase 6: Test Suite Stabilization'

View File

@@ -3,47 +3,61 @@ from unittest.mock import MagicMock, patch
import subprocess import subprocess
from shell_runner import run_powershell from shell_runner import run_powershell
def test_run_powershell_qa_callback_on_failure() -> None: def test_run_powershell_qa_callback_on_failure(vlogger) -> None:
""" """
Test that qa_callback is called when a powershell command fails (non-zero exit code). Test that qa_callback is called when a powershell command fails (non-zero exit code).
The result of the callback should be appended to the output. The result of the callback should be appended to the output.
""" """
script = "Write-Error 'something went wrong'; exit 1" script = "Write-Error 'something went wrong'; exit 1"
base_dir = "." base_dir = "."
# Mocking subprocess.run to simulate failure
mock_result = MagicMock() vlogger.log_state("Script", "N/A", script)
mock_result.stdout = ""
mock_result.stderr = "something went wrong" # Mocking subprocess.Popen
mock_result.returncode = 1 mock_process = MagicMock()
mock_process.communicate.return_value = ("", "something went wrong")
mock_process.returncode = 1
qa_callback = MagicMock(return_value="QA ANALYSIS: This looks like a syntax error.") qa_callback = MagicMock(return_value="QA ANALYSIS: This looks like a syntax error.")
with patch("subprocess.run", return_value=mock_result), \
patch("shutil.which", return_value="powershell.exe"): with patch("subprocess.Popen", return_value=mock_process), \
# We expect run_powershell to accept qa_callback patch("shutil.which", return_value="powershell.exe"):
output = run_powershell(script, base_dir, qa_callback=qa_callback) output = run_powershell(script, base_dir, qa_callback=qa_callback)
vlogger.log_state("Captured Stderr", "N/A", "something went wrong")
vlogger.log_state("QA Result", "N/A", "QA ANALYSIS: This looks like a syntax error.")
# Verify callback was called with stderr # Verify callback was called with stderr
qa_callback.assert_called_once_with("something went wrong") qa_callback.assert_called_once_with("something went wrong")
# Verify output contains the callback result # Verify output contains the callback result
assert "QA ANALYSIS: This looks like a syntax error." in output assert "QA ANALYSIS: This looks like a syntax error." in output
assert "STDERR:\nsomething went wrong" in output assert "STDERR:\nsomething went wrong" in output
assert "EXIT CODE: 1" in output assert "EXIT CODE: 1" in output
vlogger.finalize("QA Callback on Failure", "PASS", "Interceptor triggered and result appended.")
def test_run_powershell_qa_callback_on_stderr_only() -> None: def test_run_powershell_qa_callback_on_stderr_only(vlogger) -> None:
""" """
Test that qa_callback is called when a command has stderr even if exit code is 0. Test that qa_callback is called when a command has stderr even if exit code is 0.
""" """
script = "Write-Error 'non-fatal error'" script = "Write-Error 'non-fatal error'"
base_dir = "." base_dir = "."
mock_result = MagicMock()
mock_result.stdout = "Success" mock_process = MagicMock()
mock_result.stderr = "non-fatal error" mock_process.communicate.return_value = ("Success", "non-fatal error")
mock_result.returncode = 0 mock_process.returncode = 0
qa_callback = MagicMock(return_value="QA ANALYSIS: Ignorable warning.") qa_callback = MagicMock(return_value="QA ANALYSIS: Ignorable warning.")
with patch("subprocess.run", return_value=mock_result), \
patch("shutil.which", return_value="powershell.exe"): with patch("subprocess.Popen", return_value=mock_process), \
patch("shutil.which", return_value="powershell.exe"):
output = run_powershell(script, base_dir, qa_callback=qa_callback) output = run_powershell(script, base_dir, qa_callback=qa_callback)
vlogger.log_state("Stderr", "N/A", "non-fatal error")
qa_callback.assert_called_once_with("non-fatal error") qa_callback.assert_called_once_with("non-fatal error")
assert "QA ANALYSIS: Ignorable warning." in output assert "QA ANALYSIS: Ignorable warning." in output
assert "STDOUT:\nSuccess" in output assert "STDOUT:\nSuccess" in output
vlogger.finalize("QA Callback on Stderr Only", "PASS", "Interceptor triggered for non-fatal stderr.")
def test_run_powershell_no_qa_callback_on_success() -> None: def test_run_powershell_no_qa_callback_on_success() -> None:
""" """
@@ -51,13 +65,14 @@ def test_run_powershell_no_qa_callback_on_success() -> None:
""" """
script = "Write-Output 'All good'" script = "Write-Output 'All good'"
base_dir = "." base_dir = "."
mock_result = MagicMock()
mock_result.stdout = "All good" mock_process = MagicMock()
mock_result.stderr = "" mock_process.communicate.return_value = ("All good", "")
mock_result.returncode = 0 mock_process.returncode = 0
qa_callback = MagicMock() qa_callback = MagicMock()
with patch("subprocess.run", return_value=mock_result), \ with patch("subprocess.Popen", return_value=mock_process), \
patch("shutil.which", return_value="powershell.exe"): patch("shutil.which", return_value="powershell.exe"):
output = run_powershell(script, base_dir, qa_callback=qa_callback) output = run_powershell(script, base_dir, qa_callback=qa_callback)
qa_callback.assert_not_called() qa_callback.assert_not_called()
assert "STDOUT:\nAll good" in output assert "STDOUT:\nAll good" in output
@@ -70,18 +85,19 @@ def test_run_powershell_optional_qa_callback() -> None:
""" """
script = "Write-Error 'error'" script = "Write-Error 'error'"
base_dir = "." base_dir = "."
mock_result = MagicMock()
mock_result.stdout = "" mock_process = MagicMock()
mock_result.stderr = "error" mock_process.communicate.return_value = ("", "error")
mock_result.returncode = 1 mock_process.returncode = 1
with patch("subprocess.run", return_value=mock_result), \
patch("shutil.which", return_value="powershell.exe"): with patch("subprocess.Popen", return_value=mock_process), \
# Should not raise TypeError even if qa_callback is not provided patch("shutil.which", return_value="powershell.exe"):
# Should not raise TypeError even if qa_callback is not provided
output = run_powershell(script, base_dir) output = run_powershell(script, base_dir)
assert "STDERR:\nerror" in output assert "STDERR:\nerror" in output
assert "EXIT CODE: 1" in output assert "EXIT CODE: 1" in output
def test_end_to_end_tier4_integration() -> None: def test_end_to_end_tier4_integration(vlogger) -> None:
""" """
Verifies that shell_runner.run_powershell correctly uses ai_client.run_tier4_analysis. Verifies that shell_runner.run_powershell correctly uses ai_client.run_tier4_analysis.
""" """
@@ -89,17 +105,23 @@ def test_end_to_end_tier4_integration() -> None:
script = "Invoke-Item non_existent_file" script = "Invoke-Item non_existent_file"
base_dir = "." base_dir = "."
stderr_content = "Invoke-Item : Cannot find path 'C:\\non_existent_file' because it does not exist." stderr_content = "Invoke-Item : Cannot find path 'C:\\non_existent_file' because it does not exist."
mock_result = MagicMock()
mock_result.stdout = "" mock_process = MagicMock()
mock_result.stderr = stderr_content mock_process.communicate.return_value = ("", stderr_content)
mock_result.returncode = 1 mock_process.returncode = 1
expected_analysis = "Path does not exist. Verify the file path and ensure the file is present before invoking." expected_analysis = "Path does not exist. Verify the file path and ensure the file is present before invoking."
with patch("subprocess.run", return_value=mock_result), \
patch("shutil.which", return_value="powershell.exe"), \ with patch("subprocess.Popen", return_value=mock_process), \
patch("ai_client.run_tier4_analysis", return_value=expected_analysis) as mock_analysis: patch("shutil.which", return_value="powershell.exe"), \
patch("ai_client.run_tier4_analysis", return_value=expected_analysis) as mock_analysis:
vlogger.log_state("Stderr Content", "N/A", stderr_content)
output = run_powershell(script, base_dir, qa_callback=ai_client.run_tier4_analysis) output = run_powershell(script, base_dir, qa_callback=ai_client.run_tier4_analysis)
mock_analysis.assert_called_once_with(stderr_content) mock_analysis.assert_called_once_with(stderr_content)
assert f"QA ANALYSIS:\n{expected_analysis}" in output assert f"QA ANALYSIS:\n{expected_analysis}" in output
vlogger.finalize("End-to-End Tier 4 Integration", "PASS", "ai_client.run_tier4_analysis correctly called and results merged.")
def test_ai_client_passes_qa_callback() -> None: def test_ai_client_passes_qa_callback() -> None:
""" """
@@ -111,7 +133,7 @@ def test_ai_client_passes_qa_callback() -> None:
qa_callback = MagicMock(return_value="QA Analysis") qa_callback = MagicMock(return_value="QA Analysis")
# Force provider to gemini and mock its send function # Force provider to gemini and mock its send function
with patch("ai_client._provider", "gemini"), \ with patch("ai_client._provider", "gemini"), \
patch("ai_client._send_gemini", mock_send_gemini): patch("ai_client._send_gemini", mock_send_gemini):
ai_client.send( ai_client.send(
md_content="Context", md_content="Context",
user_message="Hello", user_message="Hello",
@@ -163,10 +185,10 @@ def test_gemini_provider_passes_qa_callback_to_run_script() -> None:
qa_callback = MagicMock() qa_callback = MagicMock()
# Set global state for the test # Set global state for the test
with patch("ai_client._gemini_client", mock_client), \ with patch("ai_client._gemini_client", mock_client), \
patch("ai_client._gemini_chat", None), \ patch("ai_client._gemini_chat", None), \
patch("ai_client._ensure_gemini_client"), \ patch("ai_client._ensure_gemini_client"), \
patch("ai_client._run_script", return_value="output") as mock_run_script, \ patch("ai_client._run_script", return_value="output") as mock_run_script, \
patch("ai_client._get_gemini_history_list", return_value=[]): patch("ai_client._get_gemini_history_list", return_value=[]):
# Ensure chats.create returns our mock_chat # Ensure chats.create returns our mock_chat
mock_client.chats.create.return_value = mock_chat mock_client.chats.create.return_value = mock_chat
ai_client._send_gemini( ai_client._send_gemini(