{ "track_id": "twitter_threads_extraction_20260705", "name": "Twitter/X Thread Extraction Tooling", "status": "active", "branch": "master", "created": "2026-07-05", "owner": "Tier 1 (initialized); implementation delegated to Tier 2/3.", "blocked_by": [], "blocks": [], "scope": { "new_files": [ "scripts/twitter_threads/__init__.py", "scripts/twitter_threads/error_types.py", "scripts/twitter_threads/fetch_thread.py", "scripts/twitter_threads/download_media.py", "scripts/twitter_threads/render_markdown.py", "scripts/twitter_threads/README.md", "tests/test_twitter_threads_types.py", "tests/test_twitter_threads_render.py", "tests/test_twitter_threads_media.py", "tests/test_twitter_threads_fetch.py", "docs/reports/TRACK_COMPLETION_twitter_threads_extraction_20260705.md" ], "modified_files": [], "deleted_files": [] }, "estimated_effort": { "method": "scope (per workflow.md Tier 1 Track Initialization Rules. NO day estimates.)", "phase_1": "5 tasks: scaffold + error types + dataclasses + tests + commit", "phase_2": "3 tasks: failing tests + render_markdown impl + commit", "phase_3": "3 tasks: failing tests + download_media impl + commit", "phase_4": "5 tasks: failing tests + HTML parse + URL path + CLI + commit", "phase_5": "7 tasks: README + standalone verification + smoke test + batch tests + commit + user verification + completion report" }, "verification_criteria": [ "VC1: scripts/twitter_threads/ exists with 5 files: __init__.py, error_types.py, fetch_thread.py, download_media.py, render_markdown.py", "VC2: scripts/twitter_threads/README.md exists and documents prerequisites, usage, output layout, and the copy-to-another-repo instructions", "VC3: tests/test_twitter_threads_render.py passes — given a ThreadData fixture, render_markdown.py produces the expected Markdown (YAML front-matter + per-post sections + media links)", "VC4: tests/test_twitter_threads_fetch.py passes — given a small fixture HTML file, fetch_thread.py (Strategy C path) produces the expected ThreadData", "VC5: tests/test_twitter_threads_media.py passes — download_media.py produces correctly-named files and skips existing files", "VC6: python scripts/twitter_threads/fetch_thread.py --help works (standalone invocation, no src/ imports)", "VC7: zero imports from src/, conductor/, or scripts.video_analysis in scripts/twitter_threads/ (verified via grep)", "VC8: end-to-end smoke test against the 8-thread corpus (@NOTimothyLottes + @VPCOMPRESSB) produces 8 thread.md + media/ pairs in tests/artifacts/twitter_threads_corpus/ — the acceptance corpus for the C:\\projects\\forth\\bootslop reference-generation pipeline" ], "regressions_and_pre_existing_failures": [], "pre_existing_failures_remaining": [], "deferred_to_followup_tracks": [ { "title": "Twitter thread analysis campaign (the analog of the video analysis campaign)", "description": "If the user wants to analyze many threads + cross-thread synthesis, that's a separate future track that consumes this tooling. This track ships the tooling only.", "track_status": "not yet initialized; depends on user demand" }, { "title": "RAG indexing of extracted threads", "description": "The output Markdown could be RAG-indexed for retrieval. Out of scope for this track.", "track_status": "future; depends on user demand" }, { "title": "Twitter API v2 paid-endpoint backend (Strategy D)", "description": "Documented in the README as an alternative acquisition backend; not implemented as the default.", "track_status": "future; depends on user having developer credentials" } ], "risk_register": [ { "id": "R1", "description": "gallery-dl breaks against X.com (auth wall, rate limit, or X.com API change)", "likelihood": "medium", "impact": "The URL acquisition path (Strategy A) fails; the user must fall back to the local-HTML-parse path (Strategy C)", "mitigation": "Strategy C is implemented as the fallback; the README documents the manual-export-then-parse workflow" }, { "id": "R2", "description": "X.com media CDN URLs require auth (the public-CDN assumption is wrong)", "likelihood": "low", "impact": "download_media.py fails to fetch media even after fetch_thread.py succeeds", "mitigation": "the README documents the cookies.txt workaround; download_media.py can be extended to pass cookies via urllib" }, { "id": "R3", "description": "The local-HTML-parse path (Strategy C) is brittle against X.com's HTML structure changing", "likelihood": "medium", "impact": "the offline fallback stops working", "mitigation": "the parser is stdlib-only (html.parser) and tested against fixture HTML; if X.com changes structure, the user re-exports HTML and the parser is updated" } ], "standalone_requirement": { "rationale": "Per user directive: scripts must be standalone tooling so they don't have lots of dependencies with the repo's codebase and could be utilized in others.", "enforcement": "VC7 — zero imports from src/, conductor/, or scripts.video_analysis. Verified via grep. The scripts/twitter_threads/ directory is copy-pasteable to another repo with only gallery-dl as an external dep." } }