checkpoint history segregation

aggregate.py

@@ -16,6 +16,7 @@ import re
 import glob
 from pathlib import Path, PureWindowsPath
 import summarize
+import project_manager
 
 def find_next_increment(output_dir: Path, namespace: str) -> int:
     pattern = re.compile(rf"^{re.escape(namespace)}_(\d+)\.md$")
@@ -37,14 +38,24 @@ def is_absolute_with_drive(entry: str) -> bool:
 def resolve_paths(base_dir: Path, entry: str) -> list[Path]:
     has_drive = is_absolute_with_drive(entry)
     is_wildcard = "*" in entry
+    
+    matches = []
     if is_wildcard:
         root = Path(entry) if has_drive else base_dir / entry
         matches = [Path(p) for p in glob.glob(str(root), recursive=True) if Path(p).is_file()]
-        return sorted(matches)
     else:
-        if has_drive:
-            return [Path(entry)]
-        return [(base_dir / entry).resolve()]
+        p = Path(entry) if has_drive else (base_dir / entry).resolve()
+        matches = [p]
+        
+    # Blacklist filter
+    filtered = []
+    for p in matches:
+        name = p.name.lower()
+        if name == "history.toml" or name.endswith("_history.toml"):
+            continue
+        filtered.append(p)
+        
+    return sorted(filtered)
 
 def build_discussion_section(history: list[str]) -> str:
     sections = []
@@ -214,9 +225,25 @@ def run(config: dict) -> tuple[str, Path, list[dict]]:
     return markdown, output_file, file_items
 
 def main():
-    with open("config.toml", "rb") as f:
-        import tomllib
-        config = tomllib.load(f)
+    # Load global config to find active project
+    config_path = Path("config.toml")
+    if not config_path.exists():
+        print("config.toml not found.")
+        return
+        
+    with open(config_path, "rb") as f:
+        global_cfg = tomllib.load(f)
+        
+    active_path = global_cfg.get("projects", {}).get("active")
+    if not active_path:
+        print("No active project found in config.toml.")
+        return
+        
+    # Use project_manager to load project (handles history segregation)
+    proj = project_manager.load_project(active_path)
+    # Use flat_config to make it compatible with aggregate.run()
+    config = project_manager.flat_config(proj)
+    
     markdown, output_file, _ = run(config)
     print(f"Written: {output_file}")
 

conductor/archive/history_segregation_20260224/plan.md

@@ -0,0 +1,33 @@
+# Implementation Plan: Discussion History Segregation and Blacklisting
+
+This plan follows the Test-Driven Development (TDD) workflow to move discussion history into a dedicated sibling TOML file and enforce a strict blacklist against AI agent tool access.
+
+## Phase 1: Foundation and Migration Logic
+This phase focuses on the structural changes needed to handle dual-file project configurations and the automatic migration of legacy history.
+
+- [x] Task: Research existing `ProjectManager` serialization and tool access points in `mcp_client.py`. (f400799)
+- [x] Task: Write TDD tests for migrating the `discussion` key from `manual_slop.toml` to a new sibling file. (7c18e11)
+- [x] Task: Implement automatic migration in `ProjectManager.load_project()`. (7c18e11)
+- [x] Task: Update `ProjectManager.save_project()` to persist history separately. (7c18e11)
+- [x] Task: Verify that existing history is correctly migrated and remains visible in the GUI. (ba02c8e)
+- [x] Task: Conductor - User Manual Verification 'Foundation and Migration' (Protocol in workflow.md)
+
+## Phase 2: Blacklist Enforcement
+This phase ensures the AI agent is strictly prevented from reading the history source files through its tools.
+
+- [x] Task: Write failing tests that attempt to read a known history file via the `mcp_client.py` and `aggregate.py` logic. (77f3e22)
+- [x] Task: Implement hardcoded exclusion for `*_history.toml` and `history.toml` in `mcp_client.py`. (77f3e22)
+- [x] Task: Implement hardcoded exclusion in `aggregate.py` to prevent history from being added as a raw file context. (77f3e22)
+- [x] Task: Verify that tool-based file reads for the history file return a "Permission Denied" or "Blacklisted" error. (77f3e22)
+- [x] Task: Conductor - User Manual Verification 'Blacklist Enforcement' (Protocol in workflow.md)
+
+## Phase 3: Integration and Final Validation
+This phase validates the full lifecycle, ensuring the application remains functional and secure.
+
+- [x] Task: Conduct a full walkthrough using the simulation scripts to verify history persistence across turns. (754fbe5)
+- [x] Task: Verify that the AI can still use the *curated* history provided in the prompt context but cannot access the raw file. (754fbe5)
+- [x] Task: Run full suite of automated GUI and API hook tests. (754fbe5)
+- [x] Task: Conductor - User Manual Verification 'Integration and Final Validation' (Protocol in workflow.md) [checkpoint: 754fbe5]
+
+## Phase: Review Fixes
+- [x] Task: Apply review suggestions (docstrings, annotations, import placement) (09df57d)

conductor/tracks.md

@@ -20,11 +20,6 @@ This file tracks all major tracks for the project. Each track has its own detail
 
 ---
 
-- [ ] **Track: Move discussion histories to their own toml to prevent the ai agent from reading it (will be on a blacklist).**
-*Link: [./tracks/history_segregation_20260224/](./tracks/history_segregation_20260224/)*
-
----
-
 - [ ] **Track: Update ./docs/* & ./Readme.md, review ./MainContext.md significance (should we keep it..).**
 *Link: [./tracks/documentation_refresh_20260224/](./tracks/documentation_refresh_20260224/)*
 

conductor/tracks/history_segregation_20260224/plan.md

@@ -1,30 +0,0 @@
-# Implementation Plan: Discussion History Segregation and Blacklisting
-
-This plan follows the Test-Driven Development (TDD) workflow to move discussion history into a dedicated sibling TOML file and enforce a strict blacklist against AI agent tool access.
-
-## Phase 1: Foundation and Migration Logic
-This phase focuses on the structural changes needed to handle dual-file project configurations and the automatic migration of legacy history.
-
-- [ ] Task: Research existing `ProjectManager` serialization and tool access points in `mcp_client.py`.
-- [ ] Task: Write TDD tests for migrating the `discussion` key from `manual_slop.toml` to a new sibling file.
-- [ ] Task: Implement automatic migration in `ProjectManager.load_project()`.
-- [ ] Task: Update `ProjectManager.save_project()` to persist history separately.
-- [ ] Task: Verify that existing history is correctly migrated and remains visible in the GUI.
-- [ ] Task: Conductor - User Manual Verification 'Foundation and Migration' (Protocol in workflow.md)
-
-## Phase 2: Blacklist Enforcement
-This phase ensures the AI agent is strictly prevented from reading the history source files through its tools.
-
-- [ ] Task: Write failing tests that attempt to read a known history file via the `mcp_client.py` and `aggregate.py` logic.
-- [ ] Task: Implement hardcoded exclusion for `*_history.toml` and `history.toml` in `mcp_client.py`.
-- [ ] Task: Implement hardcoded exclusion in `aggregate.py` to prevent history from being added as a raw file context.
-- [ ] Task: Verify that tool-based file reads for the history file return a "Permission Denied" or "Blacklisted" error.
-- [ ] Task: Conductor - User Manual Verification 'Blacklist Enforcement' (Protocol in workflow.md)
-
-## Phase 3: Integration and Final Validation
-This phase validates the full lifecycle, ensuring the application remains functional and secure.
-
-- [ ] Task: Conduct a full walkthrough using the simulation scripts to verify history persistence across turns.
-- [ ] Task: Verify that the AI can still use the *curated* history provided in the prompt context but cannot access the raw file.
-- [ ] Task: Run full suite of automated GUI and API hook tests.
-- [ ] Task: Conductor - User Manual Verification 'Integration and Final Validation' (Protocol in workflow.md)

mcp_client.py

@@ -87,7 +87,14 @@ def _is_allowed(path: Path) -> bool:
       - it is contained within (or equal to) one of the _base_dirs
     All paths are resolved (follows symlinks) before comparison to prevent
     symlink-based path traversal.
+    
+    CRITICAL: Blacklisted files (history) are NEVER allowed.
     """
+    # Blacklist check
+    name = path.name.lower()
+    if name == "history.toml" or name.endswith("_history.toml"):
+        return False
+
     try:
         rp = path.resolve(strict=True)
     except (OSError, ValueError):
@@ -153,11 +160,18 @@ def list_directory(path: str) -> str:
     try:
         entries = sorted(p.iterdir(), key=lambda e: (e.is_file(), e.name.lower()))
         lines = [f"Directory: {p}", ""]
+        count = 0
         for entry in entries:
+            # Blacklist check
+            name = entry.name.lower()
+            if name == "history.toml" or name.endswith("_history.toml"):
+                continue
+            
             kind = "file" if entry.is_file() else "dir "
             size = f"{entry.stat().st_size:>10,} bytes" if entry.is_file() else ""
             lines.append(f"  [{kind}]  {entry.name:<40}  {size}")
-        lines.append(f"  ({len(entries)} entries)")
+            count += 1
+        lines.append(f"  ({count} entries)")
         return "\n".join(lines)
     except Exception as e:
         return f"ERROR listing '{path}': {e}"
@@ -178,11 +192,18 @@ def search_files(path: str, pattern: str) -> str:
         if not matches:
             return f"No files matched '{pattern}' in {path}"
         lines = [f"Search '{pattern}' in {p}:", ""]
+        count = 0
         for m in matches:
+            # Blacklist check
+            name = m.name.lower()
+            if name == "history.toml" or name.endswith("_history.toml"):
+                continue
+                
             rel = m.relative_to(p)
             kind = "file" if m.is_file() else "dir "
             lines.append(f"  [{kind}]  {rel}")
-        lines.append(f"  ({len(matches)} match(es))")
+            count += 1
+        lines.append(f"  ({count} match(es))")
         return "\n".join(lines)
     except Exception as e:
         return f"ERROR searching '{path}': {e}"

project.toml

@@ -23,17 +23,3 @@ search_files = true
 get_file_summary = true
 web_search = true
 fetch_url = true
-
-[discussion]
-roles = [
-    "User",
-    "AI",
-    "Vendor API",
-    "System",
-]
-active = "main"
-
-[discussion.discussions.main]
-git_commit = ""
-last_updated = "2026-02-24T21:55:23"
-history = []

project_history.toml

@@ -0,0 +1,12 @@
+roles = [
+    "User",
+    "AI",
+    "Vendor API",
+    "System",
+]
+active = "main"
+
+[discussions.main]
+git_commit = ""
+last_updated = "2026-02-24T22:13:24"
+history = []

project_manager.py

@@ -121,14 +121,69 @@ def default_project(name: str = "unnamed") -> dict:
 
 # ── load / save ──────────────────────────────────────────────────────────────
 
-def load_project(path) -> dict:
+def get_history_path(project_path: str | Path) -> Path:
+    """Return the Path to the sibling history TOML file for a given project."""
+    p = Path(project_path)
+    return p.parent / f"{p.stem}_history.toml"
+
+
+def load_project(path: str | Path) -> dict:
+    """
+    Load a project TOML file.
+    Automatically migrates legacy 'discussion' keys to a sibling history file.
+    """
     with open(path, "rb") as f:
-        return tomllib.load(f)
+        proj = tomllib.load(f)
+    
+    # Automatic Migration: move legacy 'discussion' to sibling file
+    hist_path = get_history_path(path)
+    if "discussion" in proj:
+        disc = proj.pop("discussion")
+        # Save to history file if it doesn't exist yet (or overwrite to migrate)
+        with open(hist_path, "wb") as f:
+            tomli_w.dump(disc, f)
+        # Save the stripped project file
+        save_project(proj, path)
+        # Restore for the returned dict so GUI works as before
+        proj["discussion"] = disc
+    else:
+        # Load from sibling if it exists
+        if hist_path.exists():
+            proj["discussion"] = load_history(path)
+    
+    return proj
 
 
-def save_project(proj: dict, path):
+def load_history(project_path: str | Path) -> dict:
+    """Load the segregated discussion history from its dedicated TOML file."""
+    hist_path = get_history_path(project_path)
+    if hist_path.exists():
+        with open(hist_path, "rb") as f:
+            return tomllib.load(f)
+    return {}
+
+
+def save_project(proj: dict, path: str | Path, disc_data: dict | None = None):
+    """
+    Save the project TOML.
+    If 'discussion' is present in proj, it is moved to the sibling history file.
+    """
+    # Ensure 'discussion' is NOT in the main project dict
+    if "discussion" in proj:
+        # If disc_data wasn't provided, use the one from proj
+        if disc_data is None:
+            disc_data = proj["discussion"]
+        # Remove it so it doesn't get saved to the main file
+        proj = dict(proj) # shallow copy to avoid mutating caller's dict
+        del proj["discussion"]
+
     with open(path, "wb") as f:
         tomli_w.dump(proj, f)
+        
+    if disc_data:
+        hist_path = get_history_path(path)
+        with open(hist_path, "wb") as f:
+            tomli_w.dump(disc_data, f)
 
 
 # ── migration helper ─────────────────────────────────────────────────────────

tests/temp_project.toml

@@ -25,18 +25,3 @@ search_files = true
 get_file_summary = true
 web_search = true
 fetch_url = true
-
-[discussion]
-roles = [
-    "User",
-    "AI",
-    "Vendor API",
-    "System",
-]
-active = "main"
-auto_add = true
-
-[discussion.discussions.main]
-git_commit = ""
-last_updated = "2026-02-24T21:55:18"
-history = []

tests/temp_project_history.toml

@@ -0,0 +1,13 @@
+roles = [
+    "User",
+    "AI",
+    "Vendor API",
+    "System",
+]
+active = "main"
+auto_add = true
+
+[discussions.main]
+git_commit = ""
+last_updated = "2026-02-24T22:13:19"
+history = []

tests/test_ai_context_history.py

@@ -0,0 +1,25 @@
+import pytest
+import tomli_w
+from pathlib import Path
+import aggregate
+import project_manager
+
+def test_aggregate_includes_segregated_history(tmp_path):
+    proj_path = tmp_path / "manual_slop.toml"
+    hist_path = tmp_path / "manual_slop_history.toml"
+    
+    # Setup segregated project
+    proj_data = project_manager.default_project("test-aggregate")
+    proj_data["discussion"]["discussions"]["main"]["history"] = ["@2026-02-24T14:00:00\nUser:\nShow me history"]
+    
+    # Save (will segregate)
+    project_manager.save_project(proj_data, proj_path)
+    
+    # Run aggregate
+    loaded_proj = project_manager.load_project(proj_path)
+    config = project_manager.flat_config(loaded_proj)
+    
+    markdown, output_file, file_items = aggregate.run(config)
+    
+    assert "## Discussion History" in markdown
+    assert "Show me history" in markdown

tests/test_history_blacklist.py

@@ -0,0 +1,32 @@
+import pytest
+from pathlib import Path
+import mcp_client
+import aggregate
+
+def test_mcp_blacklist(tmp_path):
+    # Setup a "history" file
+    hist_file = tmp_path / "my_project_history.toml"
+    hist_file.write_text("secret history", encoding="utf-8")
+    
+    # Configure MCP client with the tmp_path as allowed
+    mcp_client.configure([{"path": str(hist_file)}], extra_base_dirs=[str(tmp_path)])
+    
+    # Try to read it - should fail
+    result = mcp_client.read_file(str(hist_file))
+    assert "ACCESS DENIED" in result or "BLACKLISTED" in result
+    
+    # Try to list it
+    result = mcp_client.list_directory(str(tmp_path))
+    assert "my_project_history.toml" not in result
+
+def test_aggregate_blacklist(tmp_path):
+    # Setup a "history" file
+    hist_file = tmp_path / "my_project_history.toml"
+    hist_file.write_text("secret history", encoding="utf-8")
+    
+    # Try to resolve paths including the history file
+    paths = aggregate.resolve_paths(tmp_path, "*_history.toml")
+    assert hist_file not in paths
+    
+    paths = aggregate.resolve_paths(tmp_path, "*")
+    assert hist_file not in paths

tests/test_history_migration.py

@@ -0,0 +1,56 @@
+import pytest
+import tomli_w
+import tomllib
+from pathlib import Path
+from project_manager import load_project, save_project, default_project
+
+def test_migration_on_load(tmp_path):
+    # Setup legacy project file with discussion
+    proj_path = tmp_path / "manual_slop.toml"
+    hist_path = tmp_path / "manual_slop_history.toml"
+    
+    legacy_data = default_project("test-project")
+    legacy_data["discussion"]["discussions"]["main"]["history"] = ["Hello", "World"]
+    
+    with open(proj_path, "wb") as f:
+        tomli_w.dump(legacy_data, f)
+        
+    # Load project - should trigger migration
+    loaded_data = load_project(proj_path)
+    
+    # Assertions
+    assert "discussion" in loaded_data
+    assert loaded_data["discussion"]["discussions"]["main"]["history"] == ["Hello", "World"]
+    
+    # Check that it's NOT in the main file on disk anymore
+    with open(proj_path, "rb") as f:
+        on_disk = tomllib.load(f)
+    assert "discussion" not in on_disk
+    
+    # Check history file
+    assert hist_path.exists()
+    with open(hist_path, "rb") as f:
+        hist_data = tomllib.load(f)
+    assert hist_data["discussions"]["main"]["history"] == ["Hello", "World"]
+
+def test_save_separation(tmp_path):
+    # Setup fresh project data
+    proj_path = tmp_path / "manual_slop.toml"
+    hist_path = tmp_path / "manual_slop_history.toml"
+    
+    proj_data = default_project("test-project")
+    proj_data["discussion"]["discussions"]["main"]["history"] = ["Saved", "Separately"]
+    
+    # Save project - should save both files
+    save_project(proj_data, proj_path)
+    
+    assert proj_path.exists()
+    assert hist_path.exists()
+    
+    with open(proj_path, "rb") as f:
+        p = tomllib.load(f)
+    assert "discussion" not in p
+    
+    with open(hist_path, "rb") as f:
+        h = tomllib.load(f)
+    assert h["discussions"]["main"]["history"] == ["Saved", "Separately"]

tests/test_history_persistence.py

@@ -0,0 +1,44 @@
+import pytest
+import tomli_w
+import tomllib
+from pathlib import Path
+from project_manager import load_project, save_project, default_project, entry_to_str
+
+def test_history_persistence_across_turns(tmp_path):
+    proj_path = tmp_path / "manual_slop.toml"
+    hist_path = tmp_path / "manual_slop_history.toml"
+    
+    # 1. Start project
+    proj = default_project("test-persistence")
+    save_project(proj, proj_path)
+    
+    # 2. Add a turn
+    proj = load_project(proj_path)
+    entry1 = {"role": "User", "content": "Hello", "ts": "2026-02-24T13:00:00"}
+    proj["discussion"]["discussions"]["main"]["history"].append(entry_to_str(entry1))
+    save_project(proj, proj_path)
+    
+    # Verify separation
+    with open(proj_path, "rb") as f:
+        p_disk = tomllib.load(f)
+    assert "discussion" not in p_disk
+    
+    with open(hist_path, "rb") as f:
+        h_disk = tomllib.load(f)
+    assert h_disk["discussions"]["main"]["history"] == ["@2026-02-24T13:00:00\nUser:\nHello"]
+    
+    # 3. Add another turn
+    proj = load_project(proj_path)
+    entry2 = {"role": "AI", "content": "Hi there!", "ts": "2026-02-24T13:01:00"}
+    proj["discussion"]["discussions"]["main"]["history"].append(entry_to_str(entry2))
+    save_project(proj, proj_path)
+    
+    # Verify persistence
+    with open(hist_path, "rb") as f:
+        h_disk = tomllib.load(f)
+    assert len(h_disk["discussions"]["main"]["history"]) == 2
+    assert h_disk["discussions"]["main"]["history"][1] == "@2026-02-24T13:01:00\nAI:\nHi there!"
+    
+    # 4. Reload and check
+    proj_final = load_project(proj_path)
+    assert len(proj_final["discussion"]["discussions"]["main"]["history"]) == 2