feat(gui): add Tool Usage Analytics panel with stats tracking
This commit is contained in:
@@ -145,6 +145,7 @@ class AppController:
|
|||||||
self._patch_error_message: Optional[str] = None
|
self._patch_error_message: Optional[str] = None
|
||||||
self.mma_status: str = "idle"
|
self.mma_status: str = "idle"
|
||||||
self._tool_log: List[Dict[str, Any]] = []
|
self._tool_log: List[Dict[str, Any]] = []
|
||||||
|
self._tool_stats: Dict[str, Dict[str, Any]] = {} # {tool_name: {"count": 0, "total_time_ms": 0.0, "failures": 0}}
|
||||||
self._comms_log: List[Dict[str, Any]] = []
|
self._comms_log: List[Dict[str, Any]] = []
|
||||||
self.session_usage: Dict[str, Any] = {
|
self.session_usage: Dict[str, Any] = {
|
||||||
"input_tokens": 0,
|
"input_tokens": 0,
|
||||||
@@ -1062,8 +1063,17 @@ class AppController:
|
|||||||
self._set_status("powershell done, awaiting AI...")
|
self._set_status("powershell done, awaiting AI...")
|
||||||
return output
|
return output
|
||||||
|
|
||||||
def _append_tool_log(self, script: str, result: str, source_tier: str | None = None) -> None:
|
def _append_tool_log(self, script: str, result: str, source_tier: str | None = None, elapsed_ms: float = 0.0) -> None:
|
||||||
self._tool_log.append({"script": script, "result": result, "ts": time.time(), "source_tier": source_tier})
|
self._tool_log.append({"script": script, "result": result, "ts": time.time(), "source_tier": source_tier})
|
||||||
|
tool_name = self._extract_tool_name(script)
|
||||||
|
is_failure = "REJECTED" in result or "Error" in result or "error" in result.lower()
|
||||||
|
if tool_name:
|
||||||
|
if tool_name not in self._tool_stats:
|
||||||
|
self._tool_stats[tool_name] = {"count": 0, "total_time_ms": 0.0, "failures": 0}
|
||||||
|
self._tool_stats[tool_name]["count"] += 1
|
||||||
|
self._tool_stats[tool_name]["total_time_ms"] += elapsed_ms
|
||||||
|
if is_failure:
|
||||||
|
self._tool_stats[tool_name]["failures"] += 1
|
||||||
self.ui_last_script_text = script
|
self.ui_last_script_text = script
|
||||||
self.ui_last_script_output = result
|
self.ui_last_script_output = result
|
||||||
self._trigger_script_blink = True
|
self._trigger_script_blink = True
|
||||||
@@ -1071,6 +1081,28 @@ class AppController:
|
|||||||
if self.ui_auto_scroll_tool_calls:
|
if self.ui_auto_scroll_tool_calls:
|
||||||
self._scroll_tool_calls_to_bottom = True
|
self._scroll_tool_calls_to_bottom = True
|
||||||
|
|
||||||
|
def _extract_tool_name(self, script: str) -> str:
|
||||||
|
if not script:
|
||||||
|
return "unknown"
|
||||||
|
script_lower = script.lower()
|
||||||
|
if "powershell" in script_lower or "run_powershell" in script_lower:
|
||||||
|
return "run_powershell"
|
||||||
|
if "read_file" in script_lower:
|
||||||
|
return "read_file"
|
||||||
|
if "write_file" in script_lower or "write" in script_lower:
|
||||||
|
return "write_file"
|
||||||
|
if "list_directory" in script_lower or "ls" in script_lower:
|
||||||
|
return "list_directory"
|
||||||
|
if "search_files" in script_lower or "glob" in script_lower:
|
||||||
|
return "search_files"
|
||||||
|
if "web_search" in script_lower:
|
||||||
|
return "web_search"
|
||||||
|
if "fetch_url" in script_lower:
|
||||||
|
return "fetch_url"
|
||||||
|
if "py_get" in script_lower:
|
||||||
|
return "py_get_skeleton"
|
||||||
|
return "other"
|
||||||
|
|
||||||
def resolve_pending_action(self, action_id: str, approved: bool) -> bool:
|
def resolve_pending_action(self, action_id: str, approved: bool) -> bool:
|
||||||
with self._pending_dialog_lock:
|
with self._pending_dialog_lock:
|
||||||
if action_id in self._pending_actions:
|
if action_id in self._pending_actions:
|
||||||
@@ -1160,6 +1192,7 @@ class AppController:
|
|||||||
"ai_status": self.ai_status,
|
"ai_status": self.ai_status,
|
||||||
"mma_streams": self.mma_streams,
|
"mma_streams": self.mma_streams,
|
||||||
"worker_status": self._worker_status,
|
"worker_status": self._worker_status,
|
||||||
|
"tool_stats": self._tool_stats,
|
||||||
"active_tier": self.active_tier,
|
"active_tier": self.active_tier,
|
||||||
"active_tickets": self.active_tickets,
|
"active_tickets": self.active_tickets,
|
||||||
"proposed_tracks": self.proposed_tracks
|
"proposed_tracks": self.proposed_tracks
|
||||||
@@ -1608,6 +1641,7 @@ class AppController:
|
|||||||
ai_client.reset_session()
|
ai_client.reset_session()
|
||||||
ai_client.clear_comms_log()
|
ai_client.clear_comms_log()
|
||||||
self._tool_log.clear()
|
self._tool_log.clear()
|
||||||
|
self._tool_stats.clear()
|
||||||
self._comms_log.clear()
|
self._comms_log.clear()
|
||||||
self.disc_entries.clear()
|
self.disc_entries.clear()
|
||||||
# Clear history in ALL discussions to be safe
|
# Clear history in ALL discussions to be safe
|
||||||
|
|||||||
40
src/gui_2.py
40
src/gui_2.py
@@ -333,6 +333,8 @@ class App:
|
|||||||
self._render_token_budget_panel()
|
self._render_token_budget_panel()
|
||||||
if imgui.collapsing_header("Cache Analytics"):
|
if imgui.collapsing_header("Cache Analytics"):
|
||||||
self._render_cache_panel()
|
self._render_cache_panel()
|
||||||
|
if imgui.collapsing_header("Tool Usage Analytics"):
|
||||||
|
self._render_tool_analytics_panel()
|
||||||
if imgui.collapsing_header("System Prompts"):
|
if imgui.collapsing_header("System Prompts"):
|
||||||
self._render_system_prompts_panel()
|
self._render_system_prompts_panel()
|
||||||
|
|
||||||
@@ -1484,6 +1486,44 @@ class App:
|
|||||||
if hasattr(self, '_cache_cleared_timestamp') and time.time() - self._cache_cleared_timestamp < 5:
|
if hasattr(self, '_cache_cleared_timestamp') and time.time() - self._cache_cleared_timestamp < 5:
|
||||||
imgui.text_colored(imgui.ImVec4(0.2, 1.0, 0.2, 1.0), "Cache cleared - will rebuild on next request")
|
imgui.text_colored(imgui.ImVec4(0.2, 1.0, 0.2, 1.0), "Cache cleared - will rebuild on next request")
|
||||||
|
|
||||||
|
def _render_tool_analytics_panel(self) -> None:
|
||||||
|
if not imgui.collapsing_header("Tool Usage Analytics"):
|
||||||
|
return
|
||||||
|
tool_stats = getattr(self, '_worker_status', {})
|
||||||
|
if not tool_stats:
|
||||||
|
tool_stats = {}
|
||||||
|
if hasattr(self.controller, '_tool_stats'):
|
||||||
|
tool_stats = self.controller._tool_stats
|
||||||
|
if not tool_stats:
|
||||||
|
imgui.text_disabled("No tool usage data")
|
||||||
|
return
|
||||||
|
if imgui.begin_table("tool_stats", 4, imgui.TableFlags_.borders | imgui.TableFlags_.sortable):
|
||||||
|
imgui.table_setup_column("Tool")
|
||||||
|
imgui.table_setup_column("Count")
|
||||||
|
imgui.table_setup_column("Avg (ms)")
|
||||||
|
imgui.table_setup_column("Fail %")
|
||||||
|
imgui.table_headers_row()
|
||||||
|
sorted_tools = sorted(tool_stats.items(), key=lambda x: -x[1].get("count", 0))
|
||||||
|
for tool_name, stats in sorted_tools:
|
||||||
|
count = stats.get("count", 0)
|
||||||
|
total_time = stats.get("total_time_ms", 0)
|
||||||
|
failures = stats.get("failures", 0)
|
||||||
|
avg_time = total_time / count if count > 0 else 0
|
||||||
|
fail_pct = (failures / count * 100) if count > 0 else 0
|
||||||
|
imgui.table_next_row()
|
||||||
|
imgui.table_set_column_index(0)
|
||||||
|
imgui.text(tool_name)
|
||||||
|
imgui.table_set_column_index(1)
|
||||||
|
imgui.text(str(count))
|
||||||
|
imgui.table_set_column_index(2)
|
||||||
|
imgui.text(f"{avg_time:.0f}")
|
||||||
|
imgui.table_set_column_index(3)
|
||||||
|
if fail_pct > 0:
|
||||||
|
imgui.text_colored(imgui.ImVec4(1.0, 0.2, 0.2, 1.0), f"{fail_pct:.0f}%")
|
||||||
|
else:
|
||||||
|
imgui.text("0%")
|
||||||
|
imgui.end_table()
|
||||||
|
|
||||||
def _render_message_panel(self) -> None:
|
def _render_message_panel(self) -> None:
|
||||||
# LIVE indicator
|
# LIVE indicator
|
||||||
is_live = self.ai_status in ["running powershell...", "fetching url...", "searching web...", "powershell done, awaiting AI..."]
|
is_live = self.ai_status in ["running powershell...", "fetching url...", "searching web...", "powershell done, awaiting AI..."]
|
||||||
|
|||||||
Reference in New Issue
Block a user