ed
c9135b0565
Phase 5 t5_4 (UI adaptations for 11 v2 fields): the simplest honest adaptation — render small colored badges for the 11 v2 fields where the active vendor+model supports them. Each badge has a tooltip showing the field name. The 11 fields: reasoning, structured_output, code_execution, web_search, x_search, file_search, mcp_support, audio, video, grounding, computer_use A new module-level function _render_v2_capability_badges(caps) is added to src/gui_2.py (per the HARD RULE on no new src/<thing>.py files). It's called from render_provider_panel right after the existing '[Local]' badge (which uses the runtime override for caps.local). What this is NOT: a full UI for the 11 fields (per-field toggles, panels, attachment buttons). Those are design-heavy work and need their own track. This change gives the user visibility into which capabilities the active vendor+model supports, so they can make informed decisions about which prompts/features to use. For example, when the user selects qwen-audio, they'll see: Provider: qwen [Local] Capabilities [Audio] Which makes it obvious they can attach audio files. Tests: - 2 new tests in tests/test_vendor_capabilities.py: * All 11 v2 fields are present in the helper (drift guard) * Helper is a no-op on empty caps (no fields True) - 118/118 vendor+tool+provider+import-isolation tests pass (no regressions; +2 new tests this commit) - 3 audit scripts pass
223 lines
8.4 KiB
Python
223 lines
8.4 KiB
Python
import pytest
|
|
from src.vendor_capabilities import VendorCapabilities, get_capabilities, register
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clean_registry():
|
|
import src.vendor_capabilities
|
|
snapshot = src.vendor_capabilities._REGISTRY.copy()
|
|
yield
|
|
src.vendor_capabilities._REGISTRY.clear()
|
|
src.vendor_capabilities._REGISTRY.update(snapshot)
|
|
|
|
def test_registry_lookup_known_model():
|
|
caps = VendorCapabilities(
|
|
vendor='qwen',
|
|
model='qwen-max',
|
|
vision=False,
|
|
context_window=32768
|
|
)
|
|
register(caps)
|
|
retrieved = get_capabilities('qwen', 'qwen-max')
|
|
assert retrieved.vendor == 'qwen'
|
|
assert retrieved.model == 'qwen-max'
|
|
assert retrieved.context_window == 32768
|
|
assert retrieved.vision is False
|
|
|
|
def test_fallback_to_vendor_default():
|
|
caps = VendorCapabilities(
|
|
vendor='llama',
|
|
model='*',
|
|
context_window=131072,
|
|
cost_tracking=False
|
|
)
|
|
register(caps)
|
|
retrieved = get_capabilities('llama', 'llama-3.3-future-unregistered')
|
|
assert retrieved.context_window == 131072
|
|
assert retrieved.cost_tracking is False
|
|
|
|
def test_unknown_vendor_raises():
|
|
with pytest.raises(KeyError, match='No capabilities registered'):
|
|
get_capabilities('nonexistent_vendor', 'anymodel')
|
|
|
|
V2_FIELDS: list[str] = [
|
|
'local', 'reasoning', 'structured_output', 'code_execution',
|
|
'web_search', 'x_search', 'file_search', 'mcp_support',
|
|
'audio', 'video', 'grounding', 'computer_use',
|
|
]
|
|
|
|
@pytest.mark.parametrize('field_name', V2_FIELDS)
|
|
def test_v2_field_default_is_false(field_name: str) -> None:
|
|
caps = VendorCapabilities(vendor='test', model='m')
|
|
assert getattr(caps, field_name) is False, f'{field_name} should default to False'
|
|
|
|
@pytest.mark.parametrize('field_name', V2_FIELDS)
|
|
def test_v2_field_round_trip(field_name: str) -> None:
|
|
caps = VendorCapabilities(vendor='test', model='m', **{field_name: True})
|
|
assert getattr(caps, field_name) is True, f'{field_name} should round-trip to True'
|
|
|
|
def test_v2_local_flag_works_for_local_vendor() -> None:
|
|
register(VendorCapabilities(vendor='llama', model='llama-local-test-3.1', local=True))
|
|
caps = get_capabilities('llama', 'llama-local-test-3.1')
|
|
assert caps.local is True
|
|
|
|
def test_v2_local_flag_falls_back_to_wildcard() -> None:
|
|
register(VendorCapabilities(vendor='llama', model='*', local=True))
|
|
caps = get_capabilities('llama', 'some-unregistered-model-3.1-future')
|
|
assert caps.local is True
|
|
|
|
def test_v2_local_flag_does_not_affect_other_vendors() -> None:
|
|
register(VendorCapabilities(vendor='llama', model='*', local=True))
|
|
register(VendorCapabilities(vendor='qwen', model='*'))
|
|
caps = get_capabilities('qwen', 'qwen-turbo')
|
|
assert caps.local is False
|
|
|
|
def test_runtime_caps_override_sets_local_for_llama_localhost() -> None:
|
|
from dataclasses import replace
|
|
base = VendorCapabilities(vendor='llama', model='llama-3.1-70b-versatile')
|
|
assert base.local is False
|
|
overridden = replace(base, local=True)
|
|
assert overridden.local is True
|
|
overridden2 = replace(overridden, local=False)
|
|
assert overridden2.local is False
|
|
|
|
def test_v2_per_model_population() -> None:
|
|
caps = get_capabilities('minimax', 'MiniMax-M2.5')
|
|
assert caps.reasoning is True
|
|
caps_old = get_capabilities('minimax', 'MiniMax-M2')
|
|
assert caps_old.reasoning is False
|
|
caps_grok_v = get_capabilities('grok', 'grok-2-vision')
|
|
assert caps_grok_v.web_search is True
|
|
assert caps_grok_v.x_search is True
|
|
assert caps_grok_v.vision is True
|
|
caps_qwen_audio = get_capabilities('qwen', 'qwen-audio')
|
|
assert caps_qwen_audio.audio is True
|
|
caps_qwen_long = get_capabilities('qwen', 'qwen-long')
|
|
assert caps_qwen_long.caching is True
|
|
caps_llama_reasoning = get_capabilities('llama', 'llama-3.1-405b-reasoning')
|
|
assert caps_llama_reasoning.reasoning is True
|
|
caps_llama_plain = get_capabilities('llama', 'llama-3.1-8b-instant')
|
|
assert caps_llama_plain.reasoning is False
|
|
|
|
def test_runtime_caps_override_helper_for_llama_localhost() -> None:
|
|
from src import gui_2
|
|
from src import ai_client
|
|
original_url = ai_client._llama_base_url
|
|
try:
|
|
class MockApp:
|
|
current_provider = 'llama'
|
|
mock = MockApp()
|
|
caps = VendorCapabilities(vendor='llama', model='llama-3.1-70b-versatile')
|
|
ai_client._llama_base_url = 'https://openrouter.ai/api/v1'
|
|
result = gui_2._apply_runtime_caps_override(mock, caps)
|
|
assert result.local is False
|
|
ai_client._llama_base_url = 'http://localhost:11434/v1'
|
|
result = gui_2._apply_runtime_caps_override(mock, caps)
|
|
assert result.local is True
|
|
finally:
|
|
ai_client._llama_base_url = original_url
|
|
|
|
def test_runtime_caps_override_helper_does_not_touch_other_vendors() -> None:
|
|
from src import gui_2
|
|
from src import ai_client
|
|
original_url = ai_client._llama_base_url
|
|
try:
|
|
class MockApp:
|
|
current_provider = 'qwen'
|
|
mock = MockApp()
|
|
caps = VendorCapabilities(vendor='qwen', model='qwen-turbo')
|
|
ai_client._llama_base_url = 'http://localhost:11434/v1'
|
|
result = gui_2._apply_runtime_caps_override(mock, caps)
|
|
assert result.local is False
|
|
finally:
|
|
ai_client._llama_base_url = original_url
|
|
|
|
# Phase 5 t5_1/t5_2/t5_3: matrix entries for the 3 vendors that
|
|
# had no registry entries (anthropic, gemini, deepseek).
|
|
# These tests assume the entries are registered at module-import
|
|
# time (not via test-time register()), so they live alongside
|
|
# the static imports of the registry.
|
|
|
|
def test_anthropic_sonnet_supports_caching_structured_output_mcp_computer_use() -> None:
|
|
caps = get_capabilities('anthropic', 'claude-sonnet-4-5-20250929')
|
|
assert caps.caching is True
|
|
assert caps.structured_output is True
|
|
assert caps.mcp_support is True
|
|
assert caps.computer_use is True
|
|
assert caps.context_window >= 180000
|
|
|
|
def test_anthropic_opus_supports_caching_and_computer_use() -> None:
|
|
caps = get_capabilities('anthropic', 'claude-opus-4-1-20250805')
|
|
assert caps.caching is True
|
|
assert caps.computer_use is True
|
|
assert caps.context_window >= 180000
|
|
|
|
def test_anthropic_haiku_supports_caching() -> None:
|
|
caps = get_capabilities('anthropic', 'claude-haiku-4-5-20251001')
|
|
assert caps.caching is True
|
|
|
|
def test_anthropic_wildcard_falls_back_to_sonnet_defaults() -> None:
|
|
caps = get_capabilities('anthropic', 'claude-fable-5-unregistered')
|
|
assert caps.caching is True
|
|
assert caps.structured_output is True
|
|
assert caps.mcp_support is True
|
|
assert caps.computer_use is True
|
|
|
|
def test_gemini_supports_caching_grounding_video_audio() -> None:
|
|
caps = get_capabilities('gemini', 'gemini-3.1-pro-preview')
|
|
assert caps.caching is True
|
|
assert caps.grounding is True
|
|
assert caps.video is True
|
|
assert caps.audio is True
|
|
assert caps.structured_output is True
|
|
assert caps.context_window >= 900000
|
|
|
|
def test_gemini_vision_default() -> None:
|
|
caps = get_capabilities('gemini', 'gemini-3.1-pro-preview')
|
|
assert caps.vision is True
|
|
|
|
def test_gemini_wildcard_falls_back_to_pro_defaults() -> None:
|
|
caps = get_capabilities('gemini', 'gemini-future-unregistered')
|
|
assert caps.caching is True
|
|
assert caps.grounding is True
|
|
assert caps.video is True
|
|
assert caps.audio is True
|
|
assert caps.vision is True
|
|
assert caps.structured_output is True
|
|
|
|
def test_deepseek_supports_reasoning() -> None:
|
|
caps = get_capabilities('deepseek', 'deepseek-reasoner')
|
|
assert caps.reasoning is True
|
|
assert caps.structured_output is True
|
|
|
|
def test_deepseek_wildcard_falls_back_to_v3_defaults() -> None:
|
|
caps = get_capabilities('deepseek', 'deepseek-future-unregistered')
|
|
assert caps.reasoning is True
|
|
assert caps.structured_output is True
|
|
|
|
def test_v2_capability_badge_helper_contains_all_11_v2_fields() -> None:
|
|
"""The GUI's v2 capability badges should render badges for all
|
|
11 v2 fields. This test ensures the helper stays in sync with
|
|
the v2 matrix."""
|
|
import src.gui_2
|
|
import inspect
|
|
src_lines: str = inspect.getsource(src.gui_2._render_v2_capability_badges)
|
|
known_v2_fields: list[str] = [
|
|
"reasoning", "structured_output", "code_execution",
|
|
"web_search", "x_search", "file_search", "mcp_support",
|
|
"audio", "video", "grounding", "computer_use",
|
|
]
|
|
for field in known_v2_fields:
|
|
assert f'"{field}"' in src_lines, f'v2 field {field!r} missing from _render_v2_capability_badges helper'
|
|
|
|
def test_v2_capability_badge_helper_skips_disabled_fields() -> None:
|
|
"""Sanity: a caps with all v2 fields False should produce no
|
|
badges. We can verify this by passing a default-constructed
|
|
VendorCapabilities and asserting the helper returns without
|
|
erroring. (We can't easily verify the ImGui output without
|
|
a live context, but we can verify the helper is a no-op on
|
|
the no-cap case.)"""
|
|
from src.gui_2 import _render_v2_capability_badges
|
|
from src.vendor_capabilities import VendorCapabilities
|
|
empty_caps = VendorCapabilities(vendor='test', model='empty')
|
|
_render_v2_capability_badges(empty_caps)
|