From 6152b63578201629f4c97ff16a1571531286474b Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 25 Feb 2026 21:05:00 -0500 Subject: [PATCH] chore(conductor): Checkpoint Phase 2: Manifest and Tooling for test curation track --- .../tracks/test_curation_20260225/plan.md | 10 +- run_tests.py | 122 +++++++++++++++++- tests.toml | 73 +++++++++++ 3 files changed, 199 insertions(+), 6 deletions(-) create mode 100644 tests.toml diff --git a/conductor/tracks/test_curation_20260225/plan.md b/conductor/tracks/test_curation_20260225/plan.md index 4abfce6..fec0677 100644 --- a/conductor/tracks/test_curation_20260225/plan.md +++ b/conductor/tracks/test_curation_20260225/plan.md @@ -9,10 +9,12 @@ This plan outlines the process for categorizing, organizing, and curating the ex - [x] Task: Conductor - User Manual Verification 'Phase 1: Research and Inventory' (Protocol in workflow.md) be689ad ## Phase 2: Manifest and Tooling -- [ ] Task: Design and create `tests.toml` manifest file -- [ ] Task: Implement a test runner wrapper or `pytest` configuration to respect the manifest and categories -- [ ] Task: Verify that Conductor/MMA tests can be explicitly excluded from default runs -- [ ] Task: Conductor - User Manual Verification 'Phase 2: Manifest and Tooling' (Protocol in workflow.md) +- [x] Task: T3-P2-1-STUB: Design tests.toml manifest schema (Completed by PM) +- [x] Task: T3-P2-1-IMPL: Populate tests.toml with full inventory +- [x] Task: T3-P2-2-STUB: Stub run_tests.py category-aware interface +- [x] Task: T3-P2-2-IMPL: Implement run_tests.py filtering logic (Verified) +- [x] Task: Verify that Conductor/MMA tests can be explicitly excluded from default runs (Verified) +- [x] Task: Conductor - User Manual Verification 'Phase 2: Manifest and Tooling' (Protocol in workflow.md) ## Phase 3: Curation and Consolidation - [ ] Task: Fix all identified non-redundant failing tests diff --git a/run_tests.py b/run_tests.py index 1fe1437..d2b0d5b 100644 --- a/run_tests.py +++ b/run_tests.py @@ -1,5 +1,123 @@ -import pytest +import argparse import sys +import tomllib +import pytest +from typing import Dict, List, Any + +def load_manifest(path: str) -> Dict[str, Any]: + """ + Loads a manifest file (expected to be in TOML format) from the given path. + + Args: + path: The path to the TOML manifest file. + + Returns: + A dictionary representing the loaded manifest. + + Raises: + FileNotFoundError: If the manifest file does not exist. + tomllib.TOMLDecodeError: If the manifest file is not valid TOML. + """ + try: + with open(path, 'rb') as f: + return tomllib.load(f) + except FileNotFoundError: + print(f"Error: Manifest file not found at {path}", file=sys.stderr) + raise + except tomllib.TOMLDecodeError: + print(f"Error: Could not decode TOML from {path}", file=sys.stderr) + raise + +def get_test_files(manifest: Dict[str, Any], category: str) -> List[str]: + """ + Determines the list of test files based on the manifest and a specified category. + + Args: + manifest: The loaded manifest dictionary. + category: The category of tests to retrieve. + + Returns: + A list of file paths corresponding to the tests in the given category. + Returns an empty list if the category is not found or has no tests. + """ + print(f"DEBUG: Looking for category '{category}' in manifest.", file=sys.stderr) + files = manifest.get("categories", {}).get(category, {}).get("files", []) + print(f"DEBUG: Found test files for category '{category}': {files}", file=sys.stderr) + return files + +def main(): + parser = argparse.ArgumentParser( + description="Run tests with optional manifest and category filtering, passing additional pytest arguments.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="""\ +Example usage: + python run_tests.py --manifest tests.toml --category unit -- --verbose --cov=my_module + python run_tests.py --manifest tests.toml --category integration + python run_tests.py --manifest tests.toml --category core + python run_tests.py --manifest tests.toml # Runs tests from default_categories + python run_tests.py -- --capture=no # Runs all tests with pytest args +""" + ) + parser.add_argument( + "--manifest", + type=str, + help="Path to the TOML manifest file containing test configurations." + ) + parser.add_argument( + "--category", + type=str, + help="Category of tests to run (e.g., 'unit', 'integration')." + ) + + args, remaining_pytest_args = parser.parse_known_args(sys.argv[1:]) + + selected_test_files = [] + manifest_data = None + + if args.manifest: + try: + manifest_data = load_manifest(args.manifest) + except (FileNotFoundError, tomllib.TOMLDecodeError): + # Error message already printed by load_manifest + sys.exit(1) + + if args.category: + # Case 1: --manifest and --category provided + files = get_test_files(manifest_data, args.category) + selected_test_files.extend(files) + else: + # Case 2: --manifest provided, but no --category + # Load default categories from manifest['execution']['default_categories'] + default_categories = manifest_data.get("execution", {}).get("default_categories", []) + if not default_categories: + print(f"Error: --manifest provided without --category, and no 'default_categories' found in manifest '{args.manifest}'.", file=sys.stderr) + parser.print_help(sys.stderr) + sys.exit(1) + + print(f"DEBUG: Using default categories from manifest '{args.manifest}': {default_categories}", file=sys.stderr) + for cat in default_categories: + files = get_test_files(manifest_data, cat) + selected_test_files.extend(files) + elif args.category: + # Case 3: --category provided without --manifest + print("Error: --category requires --manifest to be specified.", file=sys.stderr) + parser.print_help(sys.stderr) + sys.exit(1) + + # Combine selected test files with any remaining pytest arguments + # If --manifest was not provided, selected_test_files will be empty. + # If no tests were selected from manifest/category, selected_test_files will be empty. + pytest_command_args = selected_test_files + remaining_pytest_args + + # Filter out empty strings that might appear if remaining_pytest_args had them + final_pytest_args = [arg for arg in pytest_command_args if arg] + + # If no specific tests were selected and no manifest was provided, + # and no other pytest args were given, pytest.main([]) runs default discovery. + # This handles cases where user only passes pytest args like `python run_tests.py -- --cov=app` + # or when manifest/category selection results in an empty list and no other args are passed. + print(f"Running pytest with arguments: {final_pytest_args}", file=sys.stderr) + sys.exit(pytest.main(final_pytest_args)) if __name__ == "__main__": - sys.exit(pytest.main(sys.argv[1:])) + main() diff --git a/tests.toml b/tests.toml new file mode 100644 index 0000000..c9d1368 --- /dev/null +++ b/tests.toml @@ -0,0 +1,73 @@ +# Test Manifest for manual_slop + +[categories.core] +description = "Manual Slop Core and GUI tests" +files = [ + "tests/test_ai_context_history.py", + "tests/test_api_events.py", + "tests/test_gui_diagnostics.py", + "tests/test_gui_events.py", + "tests/test_gui_performance_requirements.py", + "tests/test_gui_stress_performance.py", + "tests/test_gui_updates.py", + "tests/test_gui2_events.py", + "tests/test_gui2_layout.py", + "tests/test_gui2_mcp.py", + "tests/test_gui2_parity.py", + "tests/test_gui2_performance.py", + "tests/test_headless_api.py", + "tests/test_headless_dependencies.py", + "tests/test_headless_startup.py", + "tests/test_history_blacklist.py", + "tests/test_history_bleed.py", + "tests/test_history_migration.py", + "tests/test_history_persistence.py", + "tests/test_history_truncation.py", + "tests/test_performance_monitor.py", + "tests/test_token_usage.py", + "tests/test_layout_reorganization.py" +] + +[categories.conductor] +description = "Conductor and MMA internal tests (Blacklisted from default core runs)" +files = [ + "tests/test_mma_exec.py", + "tests/test_mma_skeleton.py", + "tests/test_conductor_api_hook_integration.py", + "tests/conductor/test_infrastructure.py", + "tests/test_gemini_cli_adapter.py", + "tests/test_gemini_cli_integration.py", + "tests/test_ai_client_cli.py", + "tests/test_cli_tool_bridge.py", + "tests/test_gemini_metrics.py" +] + +[categories.integrations] +description = "MCP and other external integration tests" +files = [ + "tests/test_api_hook_client.py", + "tests/test_api_hook_extensions.py", + "tests/test_hooks.py", + "tests/test_sync_hooks.py", + "tests/test_mcp_perf_tool.py" +] + +[categories.simulations] +description = "AI and Workflow simulation tests" +files = [ + "tests/test_sim_ai_settings.py", + "tests/test_sim_base.py", + "tests/test_sim_context.py", + "tests/test_sim_execution.py", + "tests/test_sim_tools.py", + "tests/test_workflow_sim.py", + "tests/test_extended_sims.py", + "tests/test_user_agent.py", + "tests/test_live_workflow.py", + "tests/test_agent_capabilities.py", + "tests/test_agent_tools_wiring.py" +] + +[execution] +default_categories = ["core", "integrations", "simulations"] +blacklist_categories = ["conductor"]