test(bias): add efficacy simulation tests and enhance strategy labels
This commit is contained in:
51
tests/test_bias_efficacy.py
Normal file
51
tests/test_bias_efficacy.py
Normal file
@@ -0,0 +1,51 @@
|
||||
import pytest
|
||||
from src import ai_client
|
||||
from src.models import ToolPreset, Tool, BiasProfile
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
def test_bias_efficacy_prompt_generation():
|
||||
# Verify that the generated prompt contains the expected nudges and strategy
|
||||
preset = ToolPreset(name="BiasTest", categories={
|
||||
"General": [
|
||||
Tool(name="list_directory", weight=5),
|
||||
Tool(name="search_files", weight=1)
|
||||
]
|
||||
})
|
||||
bias = BiasProfile(name="Balanced", category_multipliers={"General": 1.0})
|
||||
|
||||
with patch("src.ai_client._active_tool_preset", preset):
|
||||
with patch("src.ai_client._active_bias_profile", bias):
|
||||
prompt = ai_client._get_combined_system_prompt()
|
||||
|
||||
# Check strategy section
|
||||
assert "Tooling Strategy" in prompt
|
||||
assert "list_directory" in prompt
|
||||
assert "PREFERRED" in prompt or "HIGH PRIORITY" in prompt
|
||||
|
||||
# Check tool definitions nudging
|
||||
with patch("src.ai_client._agent_tools", {"list_directory": True, "search_files": True}):
|
||||
# Anthropic tools as an example
|
||||
with patch("src.ai_client._CACHED_ANTHROPIC_TOOLS", None):
|
||||
tools = ai_client._get_anthropic_tools()
|
||||
|
||||
ls_tool = next(t for t in tools if t["name"] == "list_directory")
|
||||
search_tool = next(t for t in tools if t["name"] == "search_files")
|
||||
|
||||
assert "[HIGH PRIORITY]" in ls_tool["description"]
|
||||
assert "[LOW PRIORITY]" in ls_tool["description"] or "[LOW PRIORITY]" in search_tool["description"]
|
||||
|
||||
def test_bias_parameter_nudging():
|
||||
preset = ToolPreset(name="ParamTest", categories={
|
||||
"General": [
|
||||
Tool(name="read_file", parameter_bias={"path": "CRITICAL"})
|
||||
]
|
||||
})
|
||||
|
||||
with patch("src.ai_client._active_tool_preset", preset):
|
||||
with patch("src.ai_client._agent_tools", {"read_file": True}):
|
||||
with patch("src.ai_client._CACHED_ANTHROPIC_TOOLS", None):
|
||||
tools = ai_client._get_anthropic_tools()
|
||||
read_tool = next(t for t in tools if t["name"] == "read_file")
|
||||
|
||||
path_param_desc = read_tool["input_schema"]["properties"]["path"]["description"]
|
||||
assert "[CRITICAL]" in path_param_desc
|
||||
Reference in New Issue
Block a user