manual_slop/conductor/tracks/chunkification_optimization_20260608_PLACEHOLDER/metadata.json

{
  "track_id": "chunkification_optimization_20260608_PLACEHOLDER",
  "name": "Chunkification Optimization (C11 Pipeline Contingency)",
  "initialized": "2026-06-08",
  "owner": "tier2-tech-lead",
  "priority": "deferred",
  "status": "contingency (not active)",
  "type": "contingency document (no implementation plan until hard constraint surfaces)",
  "scope": {
    "new_files": [
      "conductor/tracks/chunkification_optimization_20260608_PLACEHOLDER/spec.md",
      "conductor/tracks/chunkification_optimization_20260608_PLACEHOLDER/metadata.json",
      "conductor/tracks/chunkification_optimization_20260608_PLACEHOLDER/state.toml",
      "conductor/tracks/chunkification_optimization_20260608_PLACEHOLDER/index.md"
    ],
    "modified_files": [],
    "deferred_until": "a hard constraint surfaces that no existing Python package can solve, AND the target is hot enough to justify the C11 build cost"
  },
  "blocked_by": [
    "profiling_evidence_of_hard_constraint"
  ],
  "blocks": [],
  "estimated_phases": 0,
  "spec": "spec.md",
  "plan": null,
  "activation_criteria": [
    "Profiling shows a real bottleneck in the target code path (markdown parsing OR snapshot processing OR log aggregation OR RAG indexing)",
    "The bottleneck cannot be solved with existing Python packages (markdown-it-py, pickle, msgspec, orjson, numpy, pandas, etc.)",
    "The user explicitly approves activation"
  ],
  "user_corrections_applied": [
    "v1 framing (stateful C extension) revised to v2 (request/response blob pipeline) per user: 'the python would have to define the payload in a simple text or binary format as the request and then the extension pipeline in C11 would do the ops and provide the output in another binary or text blob/s'",
    "v1 'build it now' revised to 'build only when hard constraint surfaces' per user: 'only worth it if I reach a hard constraint that I cannot solve with an existing python package'",
    "The 2 cited targets (markdown parsing, snapshot processing) are NOT currently bottlenecks per src/aggregate.py:380-454 and src/history.py:1-141. First fix if they become bottlenecks: add markdown-it-py OR switch to pickle/msgspec — NOT C11"
  ],
  "shape_when_activated": {
    "model": "subprocess-launch (NOT in-process FFI for v1)",
    "wire_format": "text envelope v1 (debuggable), binary v2 (fast), or hybrid envelope-text + payload-binary",
    "c11_api": "single entry point pipeline_run(Slice request) -> PipelineResponse",
    "python_wrapper": "subprocess.run(['./manual_slop_pipeline'], input=request, capture_output=True, text=True)",
    "build": "clang -O3 -std=c23 -shared chunks_module.c -o libchunks.so (or .dll on Windows)",
    "deploy": "single binary shipped alongside Python wheel; uv + pyproject.toml builds C binary as part of uv sync"
  },
  "verification_criteria": [
    "spec.md exists as a 1-page contingency document",
    "metadata.json declares status = 'contingency (not active)' and priority = 'deferred'",
    "state.toml declares status = 'deferred' with no implementation tasks",
    "The 4 activation criteria are explicit",
    "The 2 current-target analyses cite actual code paths (src/aggregate.py:380-454, src/history.py:1-141) and conclude 'NOT a bottleneck today'",
    "No code is being modified by this contingency",
    "Cross-references to the v2 assessment (docs/reports/c11_python_interop_assessment_20260608.md) and the original proposal (docs/reports/session_synthesis_20260608.md §8.2) are present"
  ],
  "links": {
    "report": null,
    "comparison_table": null,
    "decisions": null,
    "takeaways": null,
    "user_signal_recorded": "User explicitly said 'only worth it under hard constraint' and specified the request/response blob pipeline model. Both corrections are recorded in user_corrections_applied.",
    "related_tracks": [],
    "external": [
      "Reece's Xar: docs/transcripts/i-h95QIGchY_assuming_as_much_as_possible_andrewreece.txt §56:42",
      "User's chunk-ideation: docs/ideation/ed_chunk_data_structures_20260523.md",
      "v1+v2 assessment: docs/reports/c11_python_interop_assessment_20260608.md",
      "SSDL digest (theoretical foundation): docs/reports/computational_shapes_ssdl_digest_20260608.md (Technique 5 'Assume-away (Xar)' in §2.2 + 'Xar-style chunked arrays' in §5.2 pre-support this track; the 'Assume as much as possible' lens in §4 is the threshold-shift rationale)"
    ]
  }
}