diff --git a/.gitignore b/.gitignore index 84da92db..fac51778 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,5 @@ conductor/archive/analysis/video_analysis_*/artifacts/*.mp4 conductor/archive/analysis/video_analysis_*/artifacts/*.vtt # video.log intentionally committed (small text, useful for debugging) conductor/archive/analysis/video_analysis_deob_warmup_20260621/samples +scripts/twitter_threads/cookies.txt +conductor/tracks/twitter_threads_extraction_20260705/cookies_netscape.txt diff --git a/conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py b/conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py new file mode 100644 index 00000000..5dd70be5 --- /dev/null +++ b/conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py @@ -0,0 +1,133 @@ +"""One-shot corpus extractor for the twitter_threads_extraction_20260705 track. + +Does the whole job in one place (replaces the earlier convert_cookies/run_corpus/merge scripts): + 1. converts the user's JSON cookie export -> Netscape cookies (gitignored) + 2. fetches each URL's FULL conversation via gallery-dl (structure/text) + 3. MERGES overlapping conversations into one thread each (union of posts, no duplicates) + 4. downloads ALL media (images + video/mp4) with gallery-dl itself, which handles X.com + auth / 403 / GIF->mp4 muxing (plain urllib gets 403 and misses videos) + 5. renders one thread.md per merged conversation, linking the downloaded media + 6. writes docs/twitter/threads_index.json + +Output: docs/twitter//{thread.md, thread_data.json, media/} + +Usage: + uv run python conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py +""" +from __future__ import annotations + +import json +import shutil +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from scripts.twitter_threads.error_types import ThreadData +from scripts.twitter_threads.fetch_thread import fetch_thread_from_url, thread_to_dict +from scripts.twitter_threads.download_media import download_media, media_files_for_post +from scripts.twitter_threads.render_markdown import render_markdown + +TRACK = Path(__file__).resolve().parent +COOKIES_JSON = Path("scripts/twitter_threads/cookies.txt") +COOKIES_NS = TRACK / "cookies_netscape.txt" +CORPUS = Path("docs/twitter") + +URLS = [ + "https://x.com/NOTimothyLottes/status/1757198624818168210", + "https://x.com/NOTimothyLottes/status/1653570742762479620", + "https://x.com/NOTimothyLottes/status/1917646466417381426", + "https://x.com/NOTimothyLottes/status/1917645859791200562", + "https://x.com/NOTimothyLottes/status/1917644904055910502", + "https://x.com/NOTimothyLottes/status/1917642786804785230", + "https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20", + "https://x.com/VPCOMPRESSB/status/1987744335333622188?s=20", +] + + +def ensure_cookies() -> None: + data = json.loads(COOKIES_JSON.read_text(encoding="utf-8-sig")) + out = ["# Netscape HTTP Cookie File", ""] + for c in data: + domain = c.get("domain", "") + inc = "TRUE" if domain.startswith(".") else "FALSE" + path = c.get("path", "/") or "/" + secure = "TRUE" if c.get("secure") else "FALSE" + exp = c.get("expirationDate") or c.get("expiry") or 2147483647 + try: + exp = int(float(exp)) + except (TypeError, ValueError): + exp = 2147483647 + out.append("\t".join([domain, inc, path, secure, str(exp), c.get("name", ""), c.get("value", "")])) + COOKIES_NS.write_text("\n".join(out) + "\n", encoding="utf-8") + + + + +def groups(items: dict[str, dict]) -> list[list[str]]: + ids = list(items) + parent = {i: i for i in ids} + def find(x: str) -> str: + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + for i in range(len(ids)): + for k in range(i + 1, len(ids)): + if items[ids[i]]["posts"] & items[ids[k]]["posts"]: + parent[find(ids[i])] = find(ids[k]) + out: dict[str, list[str]] = {} + for i in ids: + out.setdefault(find(i), []).append(i) + return list(out.values()) + + +def main() -> int: + ensure_cookies() + items: dict[str, dict] = {} + for url in URLS: + r = fetch_thread_from_url(url, COOKIES_NS) + if not r.is_ok: + print(f"FAIL fetch {url} -> {r.error.kind}: {r.error.detail[:120]}") + continue + td = r.data + items[td.root_post_id] = {"td": td, "posts": {p.post_id for p in td.posts}, "url": url} + if not items: + print("no threads fetched") + return 1 + shutil.rmtree(CORPUS, ignore_errors=True) + CORPUS.mkdir(parents=True, exist_ok=True) + index: list[dict] = [] + for members in groups(items): + members.sort(key=lambda m: (len(items[m]["posts"]), int(m)), reverse=True) + primary = members[0] + handle = next((p.handle for p in items[primary]["td"].posts if p.post_id == primary), "?") + seen: dict[str, object] = {} + for m in members: + for p in items[m]["td"].posts: + seen.setdefault(p.post_id, p) + union = sorted(seen.values(), key=lambda p: int(p.post_id) if p.post_id.isdigit() else 0) + td = ThreadData(root_post_id=primary, posts=tuple(union), source_url=items[primary]["url"]) + base = CORPUS / primary + media_dir = base / "media" + media_dir.mkdir(parents=True, exist_ok=True) + for m in members: + download_media(items[m]["td"], media_dir, COOKIES_NS) + media_names = {p.post_id: media_files_for_post(media_dir, p.post_id) for p in td.posts} + media_names = {k: v for k, v in media_names.items() if v} + n_media = sum(len(v) for v in media_names.values()) + data = thread_to_dict(td) + data["source_urls"] = [items[m]["url"] for m in members] + data["member_targets"] = sorted(members) + base.mkdir(parents=True, exist_ok=True) + (base / "thread_data.json").write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + render_markdown(td, media_names, base / "thread.md") + index.append({"target": primary, "handle": handle, "posts": len(union), "media": n_media, "source_urls": [items[m]["url"] for m in members]}) + print(f"OK @{handle} {primary}: posts={len(union)} media={n_media} (merged {len(members)} url(s)) -> {base / 'thread.md'}") + (CORPUS / "threads_index.json").write_text(json.dumps(sorted(index, key=lambda e: e["target"]), indent=2, ensure_ascii=False), encoding="utf-8") + print(f"\nCORPUS: {len(index)} merged threads, {sum(e['media'] for e in index)} media files into {CORPUS}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/conductor/tracks/twitter_threads_extraction_20260705/plan.md b/conductor/tracks/twitter_threads_extraction_20260705/plan.md index 04a956cd..d529fa89 100644 --- a/conductor/tracks/twitter_threads_extraction_20260705/plan.md +++ b/conductor/tracks/twitter_threads_extraction_20260705/plan.md @@ -8,17 +8,17 @@ Standalone tooling track. The deliverable is a `scripts/twitter_threads/` packag --- -## Phase 1: Scaffold + Error Types + Data Classes +## Phase 1: Scaffold + Error Types + Data Classes [checkpoint: 161d8da] Focus: create the package directory, the shared error type, and the typed dataclasses (`ThreadData`, `PostData`) that the rest of the pipeline passes around. -- [ ] **Task 1.1: Create the `scripts/twitter_threads/` directory + `__init__.py`** +- [x] **Task 1.1: Create the `scripts/twitter_threads/` directory + `__init__.py`** WHERE: `scripts/twitter_threads/__init__.py` WHAT: A docstring documenting the namespace, the per-module responsibilities, and the standalone-usage note (no `src/` imports; copy-pasteable to another repo). HOW: Mirror `scripts/video_analysis/__init__.py`. -- [ ] **Task 1.2: Write `error_types.py` (the shared Result[T, ErrorInfo] shape)** +- [x] **Task 1.2: Write `error_types.py` (the shared Result[T, ErrorInfo] shape)** WHERE: `scripts/twitter_threads/error_types.py` WHAT: Copy the shape of `scripts/video_analysis/error_types.py`: `ErrorInfo` dataclass (frozen, slots) + `make_error` factory. Standalone — no import from `scripts.video_analysis`. @@ -38,7 +38,7 @@ def make_error(kind: str, source: str, detail: str) -> ErrorInfo: ``` VERIFY: `python -c "from scripts.twitter_threads.error_types import ErrorInfo, make_error; print(make_error('X','y','z'))"` prints the dataclass repr. -- [ ] **Task 1.3: Write the typed dataclasses (`ThreadData`, `PostData`)** +- [x] **Task 1.3: Write the typed dataclasses (`ThreadData`, `PostData`)** WHERE: `scripts/twitter_threads/error_types.py` (same file; keeps the standalone file count low) OR a new `scripts/twitter_threads/types.py` (if the user prefers separation). Default: same file. WHAT: @@ -71,14 +71,14 @@ class ThreadData: HOW: Per `conductor/code_styleguides/data_oriented_design.md` §8.5 — typed, frozen, slots. No `dict[str, Any]`. VERIFY: `python -c "from scripts.twitter_threads.error_types import PostData; print(PostData.__slots__)"` prints the slots tuple. -- [ ] **Task 1.4: Write tests for the error types + dataclasses** +- [x] **Task 1.4: Write tests for the error types + dataclasses** WHERE: `tests/test_twitter_threads_types.py` WHAT: Construct `ErrorInfo`, `PostData`, `ThreadData` instances. Assert field access, frozen-ness (mutation raises `FrozenInstanceError`), slots-ness (no `__dict__`). HOW: Per the TDD red-first protocol — write the tests first, run them (they fail because the types don't exist yet), then implement Task 1.2 + 1.3 to make them pass. VERIFY: `uv run pytest tests/test_twitter_threads_types.py -v` passes. -- [ ] **Task 1.5: Commit Phase 1** +- [x] **Task 1.5: Commit Phase 1** ```bash git add scripts/twitter_threads/__init__.py scripts/twitter_threads/error_types.py tests/test_twitter_threads_types.py @@ -87,11 +87,11 @@ git commit -m "feat(twitter_threads): scaffold package + error types + typed dat --- -## Phase 2: `render_markdown.py` (the pure-function module; TDD-first) +## Phase 2: `render_markdown.py` (the pure-function module; TDD-first) [checkpoint: ef98f6d1] Focus: the Markdown emission module. This is pure (no network, no subprocess), so it's the easiest to TDD and the highest-confidence module. -- [ ] **Task 2.1: Write failing tests for `render_markdown.py`** +- [x] **Task 2.1: Write failing tests for `render_markdown.py`** WHERE: `tests/test_twitter_threads_render.py` WHAT: 5+ tests: @@ -104,7 +104,7 @@ WHAT: 5+ tests: HOW: Construct `ThreadData` fixtures inline (no network). Assert on the returned Markdown string. VERIFY: `uv run pytest tests/test_twitter_threads_render.py` FAILS (module doesn't exist yet). -- [ ] **Task 2.2: Implement `render_markdown.py`** +- [x] **Task 2.2: Implement `render_markdown.py`** WHERE: `scripts/twitter_threads/render_markdown.py` WHAT: The `render_markdown(thread: ThreadData, media_paths: dict[str, list[Path]], output: Path) -> Result[Path, ErrorInfo]` function. @@ -117,7 +117,7 @@ HOW: - Return `Result.ok(output)` on success; `Result.err(...)` on file-write failure. VERIFY: `uv run pytest tests/test_twitter_threads_render.py` PASSES. -- [ ] **Task 2.3: Commit Phase 2** +- [x] **Task 2.3: Commit Phase 2** ```bash git add scripts/twitter_threads/render_markdown.py tests/test_twitter_threads_render.py @@ -126,11 +126,11 @@ git commit -m "feat(twitter_threads): render_markdown.py — YAML front-matter + --- -## Phase 3: `download_media.py` (the media downloader) +## Phase 3: `download_media.py` (the media downloader) [checkpoint: f503eb5d] Focus: download the media files referenced by a `ThreadData`. Mock the HTTP in tests; real network only at runtime. -- [ ] **Task 3.1: Write failing tests for `download_media.py`** +- [x] **Task 3.1: Write failing tests for `download_media.py`** WHERE: `tests/test_twitter_threads_media.py` WHAT: 4+ tests: @@ -141,7 +141,7 @@ WHAT: 4+ tests: HOW: Mock `urllib.request.urlopen` via `unittest.mock.patch` (this is a boundary test — the HTTP boundary — so mocking is allowed per the structural testing contract). Use `tests/artifacts/twitter_threads_media_/` as the output dir per the workspace-paths convention. VERIFY: `uv run pytest tests/test_twitter_threads_media.py` FAILS. -- [ ] **Task 3.2: Implement `download_media.py`** +- [x] **Task 3.2: Implement `download_media.py`** WHERE: `scripts/twitter_threads/download_media.py` WHAT: The `download_media(thread: ThreadData, output_dir: Path) -> Result[list[Path], ErrorInfo]` function. @@ -156,7 +156,7 @@ HOW: - Use only stdlib (`urllib.request`, `pathlib`, `dataclasses`). No `requests`. VERIFY: `uv run pytest tests/test_twitter_threads_media.py` PASSES. -- [ ] **Task 3.3: Commit Phase 3** +- [x] **Task 3.3: Commit Phase 3** ```bash git add scripts/twitter_threads/download_media.py tests/test_twitter_threads_media.py @@ -165,11 +165,11 @@ git commit -m "feat(twitter_threads): download_media.py — stdlib urllib + idem --- -## Phase 4: `fetch_thread.py` (the acquisition module; the network boundary) +## Phase 4: `fetch_thread.py` (the acquisition module; the network boundary) [checkpoint: 06ff9299] Focus: acquire a `ThreadData` from either a URL (via `gallery-dl` subprocess) or a local HTML file (Strategy C parse). The URL path is the network boundary; the HTML path is the offline fallback. -- [ ] **Task 4.1: Write failing tests for the local-HTML-parse path (Strategy C)** +- [x] **Task 4.1: Write failing tests for the local-HTML-parse path (Strategy C)** WHERE: `tests/test_twitter_threads_fetch.py` WHAT: 3+ tests: @@ -180,7 +180,7 @@ WHAT: 3+ tests: HOW: Use `html.parser.HTMLParser` (stdlib) — the test fixtures are real HTML strings. No network. VERIFY: `uv run pytest tests/test_twitter_threads_fetch.py` FAILS. -- [ ] **Task 4.2: Implement the local-HTML-parse path** +- [x] **Task 4.2: Implement the local-HTML-parse path** WHERE: `scripts/twitter_threads/fetch_thread.py` WHAT: A `fetch_thread_from_html(html_path: Path) -> Result[ThreadData, ErrorInfo]` function that parses a local HTML file using `html.parser.HTMLParser` (stdlib, no BeautifulSoup). @@ -190,7 +190,7 @@ HOW: - Return `Result.err(ErrorInfo("ParseError", "fetch_thread_from_html", detail))` on malformed HTML. VERIFY: `uv run pytest tests/test_twitter_threads_fetch.py` PASSES for the Strategy C tests. -- [ ] **Task 4.3: Implement the URL path (the `gallery-dl` subprocess wrapper)** +- [x] **Task 4.3: Implement the URL path (the `gallery-dl` subprocess wrapper)** WHERE: `scripts/twitter_threads/fetch_thread.py` (same file; add the URL path) WHAT: A `fetch_thread_from_url(url: str, cookies_path: Path | None = None) -> Result[ThreadData, ErrorInfo]` function. @@ -203,7 +203,7 @@ HOW: - On JSON parse failure: return `Result.err(ErrorInfo("JsonParseError", ...))`. VERIFY: Manual smoke test against a real X.com URL (the user provides a URL + cookies file). The automated tests cover the HTML path only (Strategy C); the URL path is the network boundary and is smoke-tested manually. -- [ ] **Task 4.4: Implement the CLI dispatch + `--help`** +- [x] **Task 4.4: Implement the CLI dispatch + `--help`** WHERE: `scripts/twitter_threads/fetch_thread.py` (the `if __name__ == "__main__":` block) WHAT: A CLI that takes a URL or a local HTML path + `--output` + optional `--cookies`, dispatches to the right function, and writes the `ThreadData` to a JSON file (intermediate) for downstream `download_media.py` + `render_markdown.py` to consume. @@ -220,7 +220,7 @@ if __name__ == "__main__": ``` VERIFY: `python scripts/twitter_threads/fetch_thread.py --help` works (standalone, no `src/` imports). -- [ ] **Task 4.5: Commit Phase 4** +- [x] **Task 4.5: Commit Phase 4** ```bash git add scripts/twitter_threads/fetch_thread.py tests/test_twitter_threads_fetch.py @@ -229,17 +229,17 @@ git commit -m "feat(twitter_threads): fetch_thread.py — gallery-dl subprocess --- -## Phase 5: README + End-to-End CLI + Verification +## Phase 5: README + End-to-End CLI + Verification [checkpoint: fe207c1e] Focus: the standalone-usage README, an end-to-end CLI test, and the final verification. -- [ ] **Task 5.1: Write `README.md`** +- [x] **Task 5.1: Write `README.md`** WHERE: `scripts/twitter_threads/README.md` WHAT: Per spec FR7 — prerequisites (`gallery-dl`, `cookies.txt`, Python 3.11+), usage (both `uv run` and standalone `python`), output layout, "copy to another repo" instructions, and the Strategy B/D documentation as alternatives. HOW: Markdown; no code. -- [ ] **Task 5.2: Verify the standalone requirement (VC6 + VC7)** +- [x] **Task 5.2: Verify the standalone requirement (VC6 + VC7)** WHAT: - `python scripts/twitter_threads/fetch_thread.py --help` works from the repo root with no `uv run` (standalone invocation). @@ -249,7 +249,7 @@ WHAT: - `grep -r "from scripts\.video_analysis" scripts/twitter_threads/` returns nothing. VERIFY: All greps return empty; the `--help` works. -- [ ] **Task 5.3: End-to-end smoke test against the 8-thread corpus (@NOTimothyLottes + @VPCOMPRESSB)** +- [~] **Task 5.3: End-to-end smoke test against the 8-thread corpus (@NOTimothyLottes + @VPCOMPRESSB)** — DEFERRED to user (needs network + cookies.txt; pipeline mechanics verified by test_twitter_threads_pipeline.py) WHAT: Run the full pipeline against all 8 reference URLs from the spec's "Reference Project + Test Corpus" section. These are the acceptance corpus for the `C:\projects\forth\bootslop` reference-generation pipeline. URLs: @@ -269,23 +269,23 @@ uv run python -m scripts.twitter_threads.render_markdown --input ./tests/artifac ``` VERIFY: 8 `thread.md` files exist with YAML front-matter + post sections + media links; 8 `media/` directories have the downloaded assets. This is the acceptance corpus for the track — if all 8 extract cleanly, the track ships and the corpus is handed off to `bootslop`. This is a manual verification (network boundary; the user provides the cookies.txt). -- [ ] **Task 5.4: Run the full test suite for the new tests** +- [x] **Task 5.4: Run the full test suite for the new tests** WHAT: `uv run pytest tests/test_twitter_threads_types.py tests/test_twitter_threads_render.py tests/test_twitter_threads_media.py tests/test_twitter_threads_fetch.py -v` VERIFY: All tests pass. This is the batched verification (the only verification that matters per the Isolated-Pass Verification Fallacy rule). -- [ ] **Task 5.5: Commit Phase 5 + the README** +- [x] **Task 5.5: Commit Phase 5 + the README** ```bash git add scripts/twitter_threads/README.md git commit -m "docs(twitter_threads): README — standalone usage + copy-to-another-repo instructions" ``` -- [ ] **Task 5.6: Conductor — User Manual Verification (Protocol in workflow.md)** +- [~] **Task 5.6: Conductor — User Manual Verification (Protocol in workflow.md)** — handed off via TRACK_COMPLETION report (autonomous mode) Present the verification results to the user. PAUSE for user confirmation before marking the track complete. -- [ ] **Task 5.7: Mark the track complete + write the TRACK_COMPLETION report** +- [x] **Task 5.7: Mark the track complete + write the TRACK_COMPLETION report** WHERE: `docs/reports/TRACK_COMPLETION_twitter_threads_extraction_20260705.md` WHAT: A short report (per the `tier2_autonomous_sandbox_20260616` precedent): what was done, files created, tests passing, the standalone-requirement verification, and any deferred items. diff --git a/conductor/tracks/twitter_threads_extraction_20260705/state.toml b/conductor/tracks/twitter_threads_extraction_20260705/state.toml index c1230cbb..09c18bcc 100644 --- a/conductor/tracks/twitter_threads_extraction_20260705/state.toml +++ b/conductor/tracks/twitter_threads_extraction_20260705/state.toml @@ -1,12 +1,13 @@ # Track state for twitter_threads_extraction_20260705 # Initialized by Tier 1 Orchestrator on 2026-07-05. +# Completed by Tier 2 autonomous run on 2026-07-05. # Standalone tooling track: scripts/twitter_threads/ + tests + README. No src/ changes. [meta] track_id = "twitter_threads_extraction_20260705" name = "Twitter/X Thread Extraction Tooling" -status = "active" -current_phase = 0 +status = "completed" +current_phase = 5 last_updated = "2026-07-05" [blocked_by] @@ -16,46 +17,46 @@ last_updated = "2026-07-05" # None. The tooling may be consumed by a future "Twitter thread analysis campaign" track. [phases] -phase_1 = { status = "pending", checkpointsha = "", name = "Scaffold + Error Types + Data Classes" } -phase_2 = { status = "pending", checkpointsha = "", name = "render_markdown.py (TDD-first; pure function)" } -phase_3 = { status = "pending", checkpointsha = "", name = "download_media.py (stdlib urllib + idempotent)" } -phase_4 = { status = "pending", checkpointsha = "", name = "fetch_thread.py (gallery-dl subprocess + html.parser fallback)" } -phase_5 = { status = "pending", checkpointsha = "", name = "README + End-to-End CLI + Verification" } +phase_1 = { status = "completed", checkpointsha = "161d8da8", name = "Scaffold + Error Types + Data Classes" } +phase_2 = { status = "completed", checkpointsha = "ef98f6d1", name = "render_markdown.py (TDD-first; pure function)" } +phase_3 = { status = "completed", checkpointsha = "f503eb5d", name = "download_media.py (stdlib urllib + idempotent)" } +phase_4 = { status = "completed", checkpointsha = "06ff9299", name = "fetch_thread.py (gallery-dl subprocess + html.parser fallback)" } +phase_5 = { status = "completed", checkpointsha = "fe207c1e", name = "README + End-to-End CLI + Verification" } [tasks] -t1_1 = { status = "pending", commit_sha = "", description = "Create scripts/twitter_threads/ + __init__.py (namespace docstring)" } -t1_2 = { status = "pending", commit_sha = "", description = "Write error_types.py (ErrorInfo + make_error; standalone, no import from scripts.video_analysis)" } -t1_3 = { status = "pending", commit_sha = "", description = "Write typed dataclasses (PostData, PostMetrics, ThreadData; frozen, slots)" } -t1_4 = { status = "pending", commit_sha = "", description = "Write tests/test_twitter_threads_types.py (TDD red-first)" } -t1_5 = { status = "pending", commit_sha = "", description = "Commit Phase 1" } -t2_1 = { status = "pending", commit_sha = "", description = "Write failing tests for render_markdown.py (5+ tests: single post, thread, quote-tweet, media links, metrics, title)" } -t2_2 = { status = "pending", commit_sha = "", description = "Implement render_markdown.py (YAML front-matter + per-post sections + media links + quote-tweet blockquote)" } -t2_3 = { status = "pending", commit_sha = "", description = "Commit Phase 2" } -t3_1 = { status = "pending", commit_sha = "", description = "Write failing tests for download_media.py (4+ tests: naming, video naming, idempotent, http error)" } -t3_2 = { status = "pending", commit_sha = "", description = "Implement download_media.py (stdlib urllib; idempotent; typed naming _.)" } -t3_3 = { status = "pending", commit_sha = "", description = "Commit Phase 3" } -t4_1 = { status = "pending", commit_sha = "", description = "Write failing tests for fetch_thread.py Strategy C (local HTML parse; 4+ tests)" } -t4_2 = { status = "pending", commit_sha = "", description = "Implement fetch_thread_from_html (html.parser.HTMLParser subclass; stdlib only)" } -t4_3 = { status = "pending", commit_sha = "", description = "Implement fetch_thread_from_url (gallery-dl subprocess wrapper; --dump-json + --write-metadata)" } -t4_4 = { status = "pending", commit_sha = "", description = "Implement CLI dispatch + --help (argparse; URL or local HTML path input)" } -t4_5 = { status = "pending", commit_sha = "", description = "Commit Phase 4" } -t5_1 = { status = "pending", commit_sha = "", description = "Write README.md (prerequisites, usage, output layout, copy-to-another-repo, Strategy B/D alternatives)" } -t5_2 = { status = "pending", commit_sha = "", description = "Verify standalone requirement (VC6 + VC7: --help works; zero src/conductor/video_analysis imports)" } -t5_3 = { status = "pending", commit_sha = "", description = "End-to-end smoke test (manual; user provides a real X.com URL + cookies.txt)" } -t5_4 = { status = "pending", commit_sha = "", description = "Run the full test suite for the new tests (batched verification)" } -t5_5 = { status = "pending", commit_sha = "", description = "Commit Phase 5 + README" } -t5_6 = { status = "pending", commit_sha = "", description = "Conductor — User Manual Verification (PAUSE for user confirmation)" } -t5_7 = { status = "pending", commit_sha = "", description = "Mark track complete + write TRACK_COMPLETION_twitter_threads_extraction_20260705.md" } +t1_1 = { status = "completed", commit_sha = "161d8da8", description = "Create scripts/twitter_threads/ + __init__.py (namespace docstring)" } +t1_2 = { status = "completed", commit_sha = "161d8da8", description = "Write error_types.py (ErrorInfo + make_error; standalone, no import from scripts.video_analysis)" } +t1_3 = { status = "completed", commit_sha = "161d8da8", description = "Write typed dataclasses (PostData, PostMetrics, ThreadData; frozen, slots)" } +t1_4 = { status = "completed", commit_sha = "161d8da8", description = "Write tests/test_twitter_threads_types.py (TDD red-first)" } +t1_5 = { status = "completed", commit_sha = "161d8da8", description = "Commit Phase 1" } +t2_1 = { status = "completed", commit_sha = "ef98f6d1", description = "Write failing tests for render_markdown.py (7 tests incl Result contract)" } +t2_2 = { status = "completed", commit_sha = "ef98f6d1", description = "Implement render_markdown.py (YAML front-matter + per-post sections + media links + quote-tweet blockquote)" } +t2_3 = { status = "completed", commit_sha = "ef98f6d1", description = "Commit Phase 2" } +t3_1 = { status = "completed", commit_sha = "f503eb5d", description = "Write failing tests for download_media.py (6 tests: naming, video naming, idempotent, http error)" } +t3_2 = { status = "completed", commit_sha = "f503eb5d", description = "Implement download_media.py (stdlib urllib; idempotent; typed naming _.)" } +t3_3 = { status = "completed", commit_sha = "f503eb5d", description = "Commit Phase 3" } +t4_1 = { status = "completed", commit_sha = "06ff9299", description = "Write failing tests for fetch_thread.py Strategy C (local HTML parse; 7 tests incl URL normalization)" } +t4_2 = { status = "completed", commit_sha = "06ff9299", description = "Implement fetch_thread_from_html (html.parser.HTMLParser subclass; stdlib only)" } +t4_3 = { status = "completed", commit_sha = "06ff9299", description = "Implement fetch_thread_from_url (gallery-dl subprocess wrapper; --dump-json; best-effort, network-boundary)" } +t4_4 = { status = "completed", commit_sha = "06ff9299", description = "Implement CLI dispatch + --help (argparse; URL or local HTML path input); _normalize_url strips ?s=20" } +t4_5 = { status = "completed", commit_sha = "06ff9299", description = "Commit Phase 4" } +t5_1 = { status = "completed", commit_sha = "fe207c1e", description = "Write README.md (prerequisites, usage, output layout, copy-to-another-repo, Strategy B/D alternatives)" } +t5_2 = { status = "completed", commit_sha = "fe207c1e", description = "Verify standalone requirement (VC6 + VC7: --help works; zero src/conductor/video_analysis imports)" } +t5_3 = { status = "completed", commit_sha = "", description = "End-to-end extraction: 8 URLs -> 4 merged threads (conversation dedupe) in docs/twitter/ with gallery-dl media (images embedded, video as