Compare commits

...

2 Commits

3 changed files with 76 additions and 2 deletions

View File

@@ -47,7 +47,7 @@ This file tracks all major tracks for the project. Each track has its own detail
8. [x] **Track: Cache Analytics Display** 8. [x] **Track: Cache Analytics Display**
*Link: [./tracks/cache_analytics_20260306/](./tracks/cache_analytics_20260306/)* *Link: [./tracks/cache_analytics_20260306/](./tracks/cache_analytics_20260306/)*
9. [ ] **Track: Tool Usage Analytics** 9. [x] **Track: Tool Usage Analytics**
*Link: [./tracks/tool_usage_analytics_20260306/](./tracks/tool_usage_analytics_20260306/)* *Link: [./tracks/tool_usage_analytics_20260306/](./tracks/tool_usage_analytics_20260306/)*
10. [ ] **Track: Session Insights & Efficiency Scores** 10. [ ] **Track: Session Insights & Efficiency Scores**

View File

@@ -145,6 +145,7 @@ class AppController:
self._patch_error_message: Optional[str] = None self._patch_error_message: Optional[str] = None
self.mma_status: str = "idle" self.mma_status: str = "idle"
self._tool_log: List[Dict[str, Any]] = [] self._tool_log: List[Dict[str, Any]] = []
self._tool_stats: Dict[str, Dict[str, Any]] = {} # {tool_name: {"count": 0, "total_time_ms": 0.0, "failures": 0}}
self._comms_log: List[Dict[str, Any]] = [] self._comms_log: List[Dict[str, Any]] = []
self.session_usage: Dict[str, Any] = { self.session_usage: Dict[str, Any] = {
"input_tokens": 0, "input_tokens": 0,
@@ -1062,8 +1063,17 @@ class AppController:
self._set_status("powershell done, awaiting AI...") self._set_status("powershell done, awaiting AI...")
return output return output
def _append_tool_log(self, script: str, result: str, source_tier: str | None = None) -> None: def _append_tool_log(self, script: str, result: str, source_tier: str | None = None, elapsed_ms: float = 0.0) -> None:
self._tool_log.append({"script": script, "result": result, "ts": time.time(), "source_tier": source_tier}) self._tool_log.append({"script": script, "result": result, "ts": time.time(), "source_tier": source_tier})
tool_name = self._extract_tool_name(script)
is_failure = "REJECTED" in result or "Error" in result or "error" in result.lower()
if tool_name:
if tool_name not in self._tool_stats:
self._tool_stats[tool_name] = {"count": 0, "total_time_ms": 0.0, "failures": 0}
self._tool_stats[tool_name]["count"] += 1
self._tool_stats[tool_name]["total_time_ms"] += elapsed_ms
if is_failure:
self._tool_stats[tool_name]["failures"] += 1
self.ui_last_script_text = script self.ui_last_script_text = script
self.ui_last_script_output = result self.ui_last_script_output = result
self._trigger_script_blink = True self._trigger_script_blink = True
@@ -1071,6 +1081,28 @@ class AppController:
if self.ui_auto_scroll_tool_calls: if self.ui_auto_scroll_tool_calls:
self._scroll_tool_calls_to_bottom = True self._scroll_tool_calls_to_bottom = True
def _extract_tool_name(self, script: str) -> str:
if not script:
return "unknown"
script_lower = script.lower()
if "powershell" in script_lower or "run_powershell" in script_lower:
return "run_powershell"
if "read_file" in script_lower:
return "read_file"
if "write_file" in script_lower or "write" in script_lower:
return "write_file"
if "list_directory" in script_lower or "ls" in script_lower:
return "list_directory"
if "search_files" in script_lower or "glob" in script_lower:
return "search_files"
if "web_search" in script_lower:
return "web_search"
if "fetch_url" in script_lower:
return "fetch_url"
if "py_get" in script_lower:
return "py_get_skeleton"
return "other"
def resolve_pending_action(self, action_id: str, approved: bool) -> bool: def resolve_pending_action(self, action_id: str, approved: bool) -> bool:
with self._pending_dialog_lock: with self._pending_dialog_lock:
if action_id in self._pending_actions: if action_id in self._pending_actions:
@@ -1160,6 +1192,7 @@ class AppController:
"ai_status": self.ai_status, "ai_status": self.ai_status,
"mma_streams": self.mma_streams, "mma_streams": self.mma_streams,
"worker_status": self._worker_status, "worker_status": self._worker_status,
"tool_stats": self._tool_stats,
"active_tier": self.active_tier, "active_tier": self.active_tier,
"active_tickets": self.active_tickets, "active_tickets": self.active_tickets,
"proposed_tracks": self.proposed_tracks "proposed_tracks": self.proposed_tracks
@@ -1608,6 +1641,7 @@ class AppController:
ai_client.reset_session() ai_client.reset_session()
ai_client.clear_comms_log() ai_client.clear_comms_log()
self._tool_log.clear() self._tool_log.clear()
self._tool_stats.clear()
self._comms_log.clear() self._comms_log.clear()
self.disc_entries.clear() self.disc_entries.clear()
# Clear history in ALL discussions to be safe # Clear history in ALL discussions to be safe

View File

@@ -333,6 +333,8 @@ class App:
self._render_token_budget_panel() self._render_token_budget_panel()
if imgui.collapsing_header("Cache Analytics"): if imgui.collapsing_header("Cache Analytics"):
self._render_cache_panel() self._render_cache_panel()
if imgui.collapsing_header("Tool Usage Analytics"):
self._render_tool_analytics_panel()
if imgui.collapsing_header("System Prompts"): if imgui.collapsing_header("System Prompts"):
self._render_system_prompts_panel() self._render_system_prompts_panel()
@@ -1484,6 +1486,44 @@ class App:
if hasattr(self, '_cache_cleared_timestamp') and time.time() - self._cache_cleared_timestamp < 5: if hasattr(self, '_cache_cleared_timestamp') and time.time() - self._cache_cleared_timestamp < 5:
imgui.text_colored(imgui.ImVec4(0.2, 1.0, 0.2, 1.0), "Cache cleared - will rebuild on next request") imgui.text_colored(imgui.ImVec4(0.2, 1.0, 0.2, 1.0), "Cache cleared - will rebuild on next request")
def _render_tool_analytics_panel(self) -> None:
if not imgui.collapsing_header("Tool Usage Analytics"):
return
tool_stats = getattr(self, '_worker_status', {})
if not tool_stats:
tool_stats = {}
if hasattr(self.controller, '_tool_stats'):
tool_stats = self.controller._tool_stats
if not tool_stats:
imgui.text_disabled("No tool usage data")
return
if imgui.begin_table("tool_stats", 4, imgui.TableFlags_.borders | imgui.TableFlags_.sortable):
imgui.table_setup_column("Tool")
imgui.table_setup_column("Count")
imgui.table_setup_column("Avg (ms)")
imgui.table_setup_column("Fail %")
imgui.table_headers_row()
sorted_tools = sorted(tool_stats.items(), key=lambda x: -x[1].get("count", 0))
for tool_name, stats in sorted_tools:
count = stats.get("count", 0)
total_time = stats.get("total_time_ms", 0)
failures = stats.get("failures", 0)
avg_time = total_time / count if count > 0 else 0
fail_pct = (failures / count * 100) if count > 0 else 0
imgui.table_next_row()
imgui.table_set_column_index(0)
imgui.text(tool_name)
imgui.table_set_column_index(1)
imgui.text(str(count))
imgui.table_set_column_index(2)
imgui.text(f"{avg_time:.0f}")
imgui.table_set_column_index(3)
if fail_pct > 0:
imgui.text_colored(imgui.ImVec4(1.0, 0.2, 0.2, 1.0), f"{fail_pct:.0f}%")
else:
imgui.text("0%")
imgui.end_table()
def _render_message_panel(self) -> None: def _render_message_panel(self) -> None:
# LIVE indicator # LIVE indicator
is_live = self.ai_status in ["running powershell...", "fetching url...", "searching web...", "powershell done, awaiting AI..."] is_live = self.ai_status in ["running powershell...", "fetching url...", "searching web...", "powershell done, awaiting AI..."]