manual_slop/conductor/tracks/test_sandbox_hardening_20260619/metadata.json

{
  "track_id": "test_sandbox_hardening_20260619",
  "name": "Test Sandbox Hardening",
  "created": "2026-06-19",
  "status": "spec_written",
  "blocked_by": [],
  "blocks": [],
  "priority": "A",
  "rationale": "User has lost important sample data multiple times over the past month because tests have written to top-level TOML files (manual_slop.toml, manual_slop_history.toml, personas.toml, presets.toml, tool_presets.toml, credentials.toml) at the project root. This track adds a 4-layer enforcement stack to make such writes impossible at the Python layer (default) and at the OS layer (opt-in).",
  "scope": {
    "new_files": [
      "scripts/audit_test_sandbox_violations.py",
      "scripts/run_tests_sandboxed.ps1",
      "tests/test_test_sandbox.py",
      "conductor/code_styleguides/test_sandbox.md"
    ],
    "modified_files": [
      "src/paths.py",
      "src/models.py",
      "sloppy.py",
      "tests/conftest.py",
      "pyproject.toml",
      "conductor/tech-stack.md",
      "conductor/code_styleguides/workspace_paths.md",
      "docs/guide_testing.md",
      ".gitignore"
    ],
    "deleted_files": []
  },
  "estimated_effort": {
    "method": "scope (per workflow.md Tier 1 Track Initialization Rules). NO day estimates.",
    "phase_1": "1 task: baseline pass-rate capture + verification that isolate_workspace + check_test_toml_paths work as documented",
    "phase_2": "1 audit script + 4 regression tests + 1 commit",
    "phase_3": "1 conftest fixture (Layer 1 audit hook) + 4 guard-specific regression tests + 1 commit",
    "phase_4": "isolate_workspace migration + pyproject.toml addopts + tech-stack.md note + 1 commit",
    "phase_5": "1 PowerShell wrapper (Layer 3) + 1 smoke test + 1 commit",
    "phase_6": "2 doc files updated or 1 new styleguide + 1 commit",
    "phase_7": "11-tier verification run + 1 commit (report)",
    "phase_8": "1 end-of-track report + 1 commit",
    "summary": "8 phases, ~8 commits, ~10-12 source files touched across scripts/, tests/, pyproject.toml, docs/, conductor/"
  },
  "verification_criteria": [
    "tests/test_test_sandbox.py exists and all 13 tests pass",
    "scripts/audit_test_sandbox_violations.py runs in both default and --strict modes",
    "pyproject.toml contains addopts = '--basetemp=tests/artifacts/_pytest_tmp' under [tool.pytest.ini_options]",
    "tests/conftest.py isolate_workspace no longer calls tmp_path_factory.mktemp (per workspace_paths.md); all env-var redirects point to paths inside ./tests/artifacts/",
    "src/paths.py:get_config_path() does NOT call os.environ.get('SLOP_CONFIG', ...); uses set_config_override() instead",
    "src/paths.py:set_config_override(path) exists and is callable from sloppy.py and conftest.py",
    "sloppy.py accepts --config argparse argument and calls paths.set_config_override() before importing src/",
    "tests/conftest.py parses sys.argv for --config at module body (BEFORE any src/ import); auto-defaults to tests/artifacts/_isolation_workspace_<RUN_ID>/config_overrides.toml",
    "tests/artifacts/_isolation_workspace_<RUN_ID>/config_overrides.toml is auto-generated on every pytest run",
    "conductor/code_styleguides/test_sandbox.md exists and documents the --config CLI flag + config_overrides.toml convention",
    "scripts/run_tests_sandboxed.ps1 exists, parses cleanly, and on Windows can be invoked (-WhatIf mode for dry-run)",
    "conductor/tech-stack.md has a dated note explaining the --basetemp choice",
    "conductor/code_styleguides/workspace_paths.md or new test_sandbox.md documents the 3-layer model",
    "Full test suite (11 tiers) runs to completion with no regression vs. pre-track baseline (1288 passed + 4 xdist-skipped per result_migration_small_files_20260617)",
    "No new @pytest.mark.skip markers added (per conductor/workflow.md Skip-Marker Policy + user directive)",
    "End-of-track report at docs/reports/TRACK_COMPLETION_test_sandbox_hardening_20260619.md"
  ],
  "risk_register": [
    {
      "id": "R1",
      "title": "Layer 1 audit hook breaks a test that legitimately writes outside ./tests/",
      "likelihood": "medium",
      "scope_impact": "the implementation may be larger than the spec suggests if many tests need to be migrated to tmp_path",
      "mitigation": "allowlist includes pytest --basetemp; RuntimeError includes test name so offending test is obvious; add new paths to allowlist only via explicit allowlist update"
    },
    {
      "id": "R2",
      "title": "Layer 1 audit hook slows down the test suite",
      "likelihood": "low",
      "scope_impact": "minimal",
      "mitigation": "sys.addaudithook is a thin C-level callback; overhead measured in <2% per Python docs"
    },
    {
      "id": "R3",
      "title": "Layer 4 audit flags a currently-passing test as a false positive",
      "likelihood": "medium",
      "scope_impact": "the implementation may be larger than the spec suggests if many tests need cleanup",
      "mitigation": "audit is INFORMATIONAL by default; --strict is opt-in for CI; fix offending test rather than suppress audit"
    },
    {
      "id": "R4",
      "title": "Layer 3 PowerShell wrapper breaks on a Windows version without the required privileges",
      "likelihood": "low",
      "scope_impact": "minimal",
      "mitigation": "wrapper is opt-in; default invocation stays uv run pytest; wrapper docs explain privilege requirements"
    },
    {
      "id": "R5",
      "title": "Existing tests that don't go through isolate_workspace still read real config files",
      "likelihood": "high",
      "scope_impact": "known gap, out of scope",
      "mitigation": "Reads are out of scope per the Out of Scope section; Layer 1 still blocks writes which is the user's primary concern"
    },
    {
      "id": "R7",
      "title": "Removing SLOP_CONFIG env var fallback breaks code paths that relied on it",
      "likelihood": "medium",
      "scope_impact": "the implementation may be larger than the spec suggests if many call sites need updating",
      "mitigation": "conftest.py auto-defaults to config_overrides.toml inside the test workspace; sloppy.py auto-defaults to root_dir/config.toml; the change should be transparent for any code that goes through get_config_path()"
    },
    {
      "id": "R8",
      "title": "conftest.py sys.argv parse at module body races with pytest's own argparse",
      "likelihood": "low",
      "scope_impact": "minimal",
      "mitigation": "pytest_addoption registers --config so pytest doesn't warn about unknown flag; sys.argv parse at module body is a known-safe pattern (per conductor/tracks/test_infrastructure_hardening_20260609 conftest patterns)"
    },
    {
      "id": "R6",
      "title": "pytest_configure setting _tmp_path_factory._basetemp uses a private API that changes between versions",
      "likelihood": "medium",
      "scope_impact": "minimal",
      "mitigation": "the --basetemp addopts is the primary mechanism; the _basetemp assignment is defensive only; if it breaks, addopts still works"
    }
  ],
  "architecture_reference": {
    "primary_styleguide": "conductor/code_styleguides/workspace_paths.md",
    "secondary_styleguides": [
      "conductor/code_styleguides/feature_flags.md",
      "conductor/code_styleguides/data_oriented_design.md"
    ],
    "related_tracks": [
      "conductor/archive/workspace_path_finalize_20260609/",
      "conductor/tracks/tier2_autonomous_sandbox_20260616/",
      "Test Consolidation & TOML Sandboxing (per conductor/tracks.md:395)"
    ],
    "pattern_references": [
      "scripts/audit_no_temp_writes.py (audit script pattern)",
      "scripts/tier2/run_tier2_sandboxed.ps1 (PowerShell wrapper pattern)",
      "scripts/check_test_toml_paths.py (existing static audit)"
    ]
  },
  "deferred_to_followup_tracks": [
    {
      "title": "Eliminate the remaining SLOP_* env vars (presets, credentials, etc.)",
      "description": "This track only eliminates SLOP_CONFIG. The other 7 SLOP_* env vars (SLOP_GLOBAL_PRESETS, SLOP_GLOBAL_TOOL_PRESETS, SLOP_GLOBAL_PERSONAS, SLOP_GLOBAL_WORKSPACE_PROFILES, SLOP_CREDENTIALS, SLOP_MCP_ENV, SLOP_LOGS_DIR, SLOP_SCRIPTS_DIR) remain env-var-driven. Per user directive, this is the 'mess' to address in follow-up tracks. Same pattern: paths.set_<thing>_override() module-level + CLI flag at entry point.",
      "track_status": "not yet specced"
    },
    {
      "title": "Read-side isolation (block reads of real config from tests)",
      "description": "Layer 1 only blocks writes; reads of real credentials.toml / config.toml still happen for tests that don't go through isolate_workspace. Future track could block reads via a stricter allowlist.",
      "track_status": "not yet specced"
    },
    {
      "title": "macOS/Linux OS-level sandbox wrapper",
      "description": "Layer 3 is Windows-only (restricted token + Job Object). A run_tests_sandboxed.sh using bwrap/unshare would extend to macOS/Linux.",
      "track_status": "not yet specced"
    },
    {
      "title": "Per-fixture sandbox strictness tuning",
      "description": "The blanket autouse fixture is the v1. A future track could add @pytest.fixture(sandbox_strict=True) for tests that need full OS isolation vs. the default Python guard.",
      "track_status": "not yet specced"
    }
  ],
  "regressions_and_pre_existing_failures": [],
  "pre_existing_failures_remaining": [],
  "user_directives": [
    "Hard sandbox for tests, similar to Tier 2 - completely banned from accessing files outside ./tests/",
    "No new @pytest.mark.skip markers",
    "User has lost important sample data multiple times - this is the primary motivation",
    "NO ENV VARS for config path. Use --config CLI flag at the entry point (sloppy.py for production, conftest.py for tests)",
    "Test workspace file naming: config_overrides.toml (per user direction)",
    "Out of scope: converting the other SLOP_* env vars (presets, credentials, etc.) to CLI flags. User considers them a separate mess to address in follow-up tracks.",
    "Hard fail on any sandbox violation (no warnings, no soft fails)",
    "Tests should never need AppData temp (tempfile.mkdtemp/mkstemp without dir= is a flag)"
  ]
}