ed/manual_slop
Private
Public Access
0
0
Fork 0
Code Activity

Merge remote-tracking branch 'tier2-clone/tier2/twitter_threads_extraction_20260705'

Browse Source
This commit is contained in:
ed
2026-07-05 19:34:31 -04:00
parent ba1dbd326f d4e809ba63
commit 9a308c36dc
58 changed files with 2789 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py
+133
View File
@@ -0,0 +1,133 @@
"""One-shot corpus extractor for the twitter_threads_extraction_20260705 track.
Does the whole job in one place (replaces the earlier convert_cookies/run_corpus/merge scripts):
1. converts the user's JSON cookie export -> Netscape cookies (gitignored)
2. fetches each URL's FULL conversation via gallery-dl (structure/text)
3. MERGES overlapping conversations into one thread each (union of posts, no duplicates)
4. downloads ALL media (images + video/mp4) with gallery-dl itself, which handles X.com
auth / 403 / GIF->mp4 muxing (plain urllib gets 403 and misses videos)
5. renders one thread.md per merged conversation, linking the downloaded media
6. writes docs/twitter/threads_index.json
Output: docs/twitter/<deepest_target_id>/{thread.md, thread_data.json, media/}
Usage:
uv run python conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py
"""
from __future__ import annotations
import json
import shutil
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
from scripts.twitter_threads.error_types import ThreadData
from scripts.twitter_threads.fetch_thread import fetch_thread_from_url, thread_to_dict
from scripts.twitter_threads.download_media import download_media, media_files_for_post
from scripts.twitter_threads.render_markdown import render_markdown
TRACK = Path(__file__).resolve().parent
COOKIES_JSON = Path("scripts/twitter_threads/cookies.txt")
COOKIES_NS = TRACK / "cookies_netscape.txt"
CORPUS = Path("docs/twitter")
URLS = [
"https://x.com/NOTimothyLottes/status/1757198624818168210",
"https://x.com/NOTimothyLottes/status/1653570742762479620",
"https://x.com/NOTimothyLottes/status/1917646466417381426",
"https://x.com/NOTimothyLottes/status/1917645859791200562",
"https://x.com/NOTimothyLottes/status/1917644904055910502",
"https://x.com/NOTimothyLottes/status/1917642786804785230",
"https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20",
"https://x.com/VPCOMPRESSB/status/1987744335333622188?s=20",
]
def ensure_cookies() -> None:
data = json.loads(COOKIES_JSON.read_text(encoding="utf-8-sig"))
out = ["# Netscape HTTP Cookie File", ""]
for c in data:
domain = c.get("domain", "")
inc = "TRUE" if domain.startswith(".") else "FALSE"
path = c.get("path", "/") or "/"
secure = "TRUE" if c.get("secure") else "FALSE"
exp = c.get("expirationDate") or c.get("expiry") or 2147483647
try:
exp = int(float(exp))
except (TypeError, ValueError):
exp = 2147483647
out.append("\t".join([domain, inc, path, secure, str(exp), c.get("name", ""), c.get("value", "")]))
COOKIES_NS.write_text("\n".join(out) + "\n", encoding="utf-8")
def groups(items: dict[str, dict]) -> list[list[str]]:
ids = list(items)
parent = {i: i for i in ids}
def find(x: str) -> str:
while parent[x] != x:
parent[x] = parent[parent[x]]
x = parent[x]
return x
for i in range(len(ids)):
for k in range(i + 1, len(ids)):
if items[ids[i]]["posts"] & items[ids[k]]["posts"]:
parent[find(ids[i])] = find(ids[k])
out: dict[str, list[str]] = {}
for i in ids:
out.setdefault(find(i), []).append(i)
return list(out.values())
def main() -> int:
ensure_cookies()
items: dict[str, dict] = {}
for url in URLS:
r = fetch_thread_from_url(url, COOKIES_NS)
if not r.is_ok:
print(f"FAIL fetch {url} -> {r.error.kind}: {r.error.detail[:120]}")
continue
td = r.data
items[td.root_post_id] = {"td": td, "posts": {p.post_id for p in td.posts}, "url": url}
if not items:
print("no threads fetched")
return 1
shutil.rmtree(CORPUS, ignore_errors=True)
CORPUS.mkdir(parents=True, exist_ok=True)
index: list[dict] = []
for members in groups(items):
members.sort(key=lambda m: (len(items[m]["posts"]), int(m)), reverse=True)
primary = members[0]
handle = next((p.handle for p in items[primary]["td"].posts if p.post_id == primary), "?")
seen: dict[str, object] = {}
for m in members:
for p in items[m]["td"].posts:
seen.setdefault(p.post_id, p)
union = sorted(seen.values(), key=lambda p: int(p.post_id) if p.post_id.isdigit() else 0)
td = ThreadData(root_post_id=primary, posts=tuple(union), source_url=items[primary]["url"])
base = CORPUS / primary
media_dir = base / "media"
media_dir.mkdir(parents=True, exist_ok=True)
for m in members:
download_media(items[m]["td"], media_dir, COOKIES_NS)
media_names = {p.post_id: media_files_for_post(media_dir, p.post_id) for p in td.posts}
media_names = {k: v for k, v in media_names.items() if v}
n_media = sum(len(v) for v in media_names.values())
data = thread_to_dict(td)
data["source_urls"] = [items[m]["url"] for m in members]
data["member_targets"] = sorted(members)
base.mkdir(parents=True, exist_ok=True)
(base / "thread_data.json").write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
render_markdown(td, media_names, base / "thread.md")
index.append({"target": primary, "handle": handle, "posts": len(union), "media": n_media, "source_urls": [items[m]["url"] for m in members]})
print(f"OK @{handle} {primary}: posts={len(union)} media={n_media} (merged {len(members)} url(s)) -> {base / 'thread.md'}")
(CORPUS / "threads_index.json").write_text(json.dumps(sorted(index, key=lambda e: e["target"]), indent=2, ensure_ascii=False), encoding="utf-8")
print(f"\nCORPUS: {len(index)} merged threads, {sum(e['media'] for e in index)} media files into {CORPUS}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
conductor/tracks/twitter_threads_extraction_20260705/plan.md
+28 -28
View File
@@ -8,17 +8,17 @@ Standalone tooling track. The deliverable is a `scripts/twitter_threads/` packag
---
## Phase 1: Scaffold + Error Types + Data Classes
## Phase 1: Scaffold + Error Types + Data Classes [checkpoint: 161d8da]
Focus: create the package directory, the shared error type, and the typed dataclasses (`ThreadData`, `PostData`) that the rest of the pipeline passes around.
- [ ] **Task 1.1: Create the `scripts/twitter_threads/` directory + `__init__.py`**
- [x] **Task 1.1: Create the `scripts/twitter_threads/` directory + `__init__.py`**
WHERE: `scripts/twitter_threads/__init__.py`
WHAT: A docstring documenting the namespace, the per-module responsibilities, and the standalone-usage note (no `src/` imports; copy-pasteable to another repo).
HOW: Mirror `scripts/video_analysis/__init__.py`.
- [ ] **Task 1.2: Write `error_types.py` (the shared Result[T, ErrorInfo] shape)**
- [x] **Task 1.2: Write `error_types.py` (the shared Result[T, ErrorInfo] shape)**
WHERE: `scripts/twitter_threads/error_types.py`
WHAT: Copy the shape of `scripts/video_analysis/error_types.py`: `ErrorInfo` dataclass (frozen, slots) + `make_error` factory. Standalone — no import from `scripts.video_analysis`.
@@ -38,7 +38,7 @@ def make_error(kind: str, source: str, detail: str) -> ErrorInfo:
```
VERIFY: `python -c "from scripts.twitter_threads.error_types import ErrorInfo, make_error; print(make_error('X','y','z'))"` prints the dataclass repr.
- [ ] **Task 1.3: Write the typed dataclasses (`ThreadData`, `PostData`)**
- [x] **Task 1.3: Write the typed dataclasses (`ThreadData`, `PostData`)**
WHERE: `scripts/twitter_threads/error_types.py` (same file; keeps the standalone file count low) OR a new `scripts/twitter_threads/types.py` (if the user prefers separation). Default: same file.
WHAT:
@@ -71,14 +71,14 @@ class ThreadData:
HOW: Per `conductor/code_styleguides/data_oriented_design.md` §8.5 — typed, frozen, slots. No `dict[str, Any]`.
VERIFY: `python -c "from scripts.twitter_threads.error_types import PostData; print(PostData.__slots__)"` prints the slots tuple.
- [ ] **Task 1.4: Write tests for the error types + dataclasses**
- [x] **Task 1.4: Write tests for the error types + dataclasses**
WHERE: `tests/test_twitter_threads_types.py`
WHAT: Construct `ErrorInfo`, `PostData`, `ThreadData` instances. Assert field access, frozen-ness (mutation raises `FrozenInstanceError`), slots-ness (no `__dict__`).
HOW: Per the TDD red-first protocol — write the tests first, run them (they fail because the types don't exist yet), then implement Task 1.2 + 1.3 to make them pass.
VERIFY: `uv run pytest tests/test_twitter_threads_types.py -v` passes.
- [ ] **Task 1.5: Commit Phase 1**
- [x] **Task 1.5: Commit Phase 1**
```bash
git add scripts/twitter_threads/__init__.py scripts/twitter_threads/error_types.py tests/test_twitter_threads_types.py
@@ -87,11 +87,11 @@ git commit -m "feat(twitter_threads): scaffold package + error types + typed dat
---
## Phase 2: `render_markdown.py` (the pure-function module; TDD-first)
## Phase 2: `render_markdown.py` (the pure-function module; TDD-first) [checkpoint: ef98f6d1]
Focus: the Markdown emission module. This is pure (no network, no subprocess), so it's the easiest to TDD and the highest-confidence module.
- [ ] **Task 2.1: Write failing tests for `render_markdown.py`**
- [x] **Task 2.1: Write failing tests for `render_markdown.py`**
WHERE: `tests/test_twitter_threads_render.py`
WHAT: 5+ tests:
@@ -104,7 +104,7 @@ WHAT: 5+ tests:
HOW: Construct `ThreadData` fixtures inline (no network). Assert on the returned Markdown string.
VERIFY: `uv run pytest tests/test_twitter_threads_render.py` FAILS (module doesn't exist yet).
- [ ] **Task 2.2: Implement `render_markdown.py`**
- [x] **Task 2.2: Implement `render_markdown.py`**
WHERE: `scripts/twitter_threads/render_markdown.py`
WHAT: The `render_markdown(thread: ThreadData, media_paths: dict[str, list[Path]], output: Path) -> Result[Path, ErrorInfo]` function.
@@ -117,7 +117,7 @@ HOW:
- Return `Result.ok(output)` on success; `Result.err(...)` on file-write failure.
VERIFY: `uv run pytest tests/test_twitter_threads_render.py` PASSES.
- [ ] **Task 2.3: Commit Phase 2**
- [x] **Task 2.3: Commit Phase 2**
```bash
git add scripts/twitter_threads/render_markdown.py tests/test_twitter_threads_render.py
@@ -126,11 +126,11 @@ git commit -m "feat(twitter_threads): render_markdown.py — YAML front-matter +
---
## Phase 3: `download_media.py` (the media downloader)
## Phase 3: `download_media.py` (the media downloader) [checkpoint: f503eb5d]
Focus: download the media files referenced by a `ThreadData`. Mock the HTTP in tests; real network only at runtime.
- [ ] **Task 3.1: Write failing tests for `download_media.py`**
- [x] **Task 3.1: Write failing tests for `download_media.py`**
WHERE: `tests/test_twitter_threads_media.py`
WHAT: 4+ tests:
@@ -141,7 +141,7 @@ WHAT: 4+ tests:
HOW: Mock `urllib.request.urlopen` via `unittest.mock.patch` (this is a boundary test — the HTTP boundary — so mocking is allowed per the structural testing contract). Use `tests/artifacts/twitter_threads_media_<test_name>/` as the output dir per the workspace-paths convention.
VERIFY: `uv run pytest tests/test_twitter_threads_media.py` FAILS.
- [ ] **Task 3.2: Implement `download_media.py`**
- [x] **Task 3.2: Implement `download_media.py`**
WHERE: `scripts/twitter_threads/download_media.py`
WHAT: The `download_media(thread: ThreadData, output_dir: Path) -> Result[list[Path], ErrorInfo]` function.
@@ -156,7 +156,7 @@ HOW:
- Use only stdlib (`urllib.request`, `pathlib`, `dataclasses`). No `requests`.
VERIFY: `uv run pytest tests/test_twitter_threads_media.py` PASSES.
- [ ] **Task 3.3: Commit Phase 3**
- [x] **Task 3.3: Commit Phase 3**
```bash
git add scripts/twitter_threads/download_media.py tests/test_twitter_threads_media.py
@@ -165,11 +165,11 @@ git commit -m "feat(twitter_threads): download_media.py — stdlib urllib + idem
---
## Phase 4: `fetch_thread.py` (the acquisition module; the network boundary)
## Phase 4: `fetch_thread.py` (the acquisition module; the network boundary) [checkpoint: 06ff9299]
Focus: acquire a `ThreadData` from either a URL (via `gallery-dl` subprocess) or a local HTML file (Strategy C parse). The URL path is the network boundary; the HTML path is the offline fallback.
- [ ] **Task 4.1: Write failing tests for the local-HTML-parse path (Strategy C)**
- [x] **Task 4.1: Write failing tests for the local-HTML-parse path (Strategy C)**
WHERE: `tests/test_twitter_threads_fetch.py`
WHAT: 3+ tests:
@@ -180,7 +180,7 @@ WHAT: 3+ tests:
HOW: Use `html.parser.HTMLParser` (stdlib) — the test fixtures are real HTML strings. No network.
VERIFY: `uv run pytest tests/test_twitter_threads_fetch.py` FAILS.
- [ ] **Task 4.2: Implement the local-HTML-parse path**
- [x] **Task 4.2: Implement the local-HTML-parse path**
WHERE: `scripts/twitter_threads/fetch_thread.py`
WHAT: A `fetch_thread_from_html(html_path: Path) -> Result[ThreadData, ErrorInfo]` function that parses a local HTML file using `html.parser.HTMLParser` (stdlib, no BeautifulSoup).
@@ -190,7 +190,7 @@ HOW:
- Return `Result.err(ErrorInfo("ParseError", "fetch_thread_from_html", detail))` on malformed HTML.
VERIFY: `uv run pytest tests/test_twitter_threads_fetch.py` PASSES for the Strategy C tests.
- [ ] **Task 4.3: Implement the URL path (the `gallery-dl` subprocess wrapper)**
- [x] **Task 4.3: Implement the URL path (the `gallery-dl` subprocess wrapper)**
WHERE: `scripts/twitter_threads/fetch_thread.py` (same file; add the URL path)
WHAT: A `fetch_thread_from_url(url: str, cookies_path: Path | None = None) -> Result[ThreadData, ErrorInfo]` function.
@@ -203,7 +203,7 @@ HOW:
- On JSON parse failure: return `Result.err(ErrorInfo("JsonParseError", ...))`.
VERIFY: Manual smoke test against a real X.com URL (the user provides a URL + cookies file). The automated tests cover the HTML path only (Strategy C); the URL path is the network boundary and is smoke-tested manually.
- [ ] **Task 4.4: Implement the CLI dispatch + `--help`**
- [x] **Task 4.4: Implement the CLI dispatch + `--help`**
WHERE: `scripts/twitter_threads/fetch_thread.py` (the `if __name__ == "__main__":` block)
WHAT: A CLI that takes a URL or a local HTML path + `--output` + optional `--cookies`, dispatches to the right function, and writes the `ThreadData` to a JSON file (intermediate) for downstream `download_media.py` + `render_markdown.py` to consume.
@@ -220,7 +220,7 @@ if __name__ == "__main__":
```
VERIFY: `python scripts/twitter_threads/fetch_thread.py --help` works (standalone, no `src/` imports).
- [ ] **Task 4.5: Commit Phase 4**
- [x] **Task 4.5: Commit Phase 4**
```bash
git add scripts/twitter_threads/fetch_thread.py tests/test_twitter_threads_fetch.py
@@ -229,17 +229,17 @@ git commit -m "feat(twitter_threads): fetch_thread.py — gallery-dl subprocess
---
## Phase 5: README + End-to-End CLI + Verification
## Phase 5: README + End-to-End CLI + Verification [checkpoint: fe207c1e]
Focus: the standalone-usage README, an end-to-end CLI test, and the final verification.
- [ ] **Task 5.1: Write `README.md`**
- [x] **Task 5.1: Write `README.md`**
WHERE: `scripts/twitter_threads/README.md`
WHAT: Per spec FR7 — prerequisites (`gallery-dl`, `cookies.txt`, Python 3.11+), usage (both `uv run` and standalone `python`), output layout, "copy to another repo" instructions, and the Strategy B/D documentation as alternatives.
HOW: Markdown; no code.
- [ ] **Task 5.2: Verify the standalone requirement (VC6 + VC7)**
- [x] **Task 5.2: Verify the standalone requirement (VC6 + VC7)**
WHAT:
- `python scripts/twitter_threads/fetch_thread.py --help` works from the repo root with no `uv run` (standalone invocation).
@@ -249,7 +249,7 @@ WHAT:
- `grep -r "from scripts\.video_analysis" scripts/twitter_threads/` returns nothing.
VERIFY: All greps return empty; the `--help` works.
- [ ] **Task 5.3: End-to-end smoke test against the 8-thread corpus (@NOTimothyLottes + @VPCOMPRESSB)**
- [~] **Task 5.3: End-to-end smoke test against the 8-thread corpus (@NOTimothyLottes + @VPCOMPRESSB)** — DEFERRED to user (needs network + cookies.txt; pipeline mechanics verified by test_twitter_threads_pipeline.py)
WHAT: Run the full pipeline against all 8 reference URLs from the spec's "Reference Project + Test Corpus" section. These are the acceptance corpus for the `C:\projects\forth\bootslop` reference-generation pipeline.
URLs:
@@ -269,23 +269,23 @@ uv run python -m scripts.twitter_threads.render_markdown --input ./tests/artifac
```
VERIFY: 8 `thread.md` files exist with YAML front-matter + post sections + media links; 8 `media/` directories have the downloaded assets. This is the acceptance corpus for the track — if all 8 extract cleanly, the track ships and the corpus is handed off to `bootslop`. This is a manual verification (network boundary; the user provides the cookies.txt).
- [ ] **Task 5.4: Run the full test suite for the new tests**
- [x] **Task 5.4: Run the full test suite for the new tests**
WHAT: `uv run pytest tests/test_twitter_threads_types.py tests/test_twitter_threads_render.py tests/test_twitter_threads_media.py tests/test_twitter_threads_fetch.py -v`
VERIFY: All tests pass. This is the batched verification (the only verification that matters per the Isolated-Pass Verification Fallacy rule).
- [ ] **Task 5.5: Commit Phase 5 + the README**
- [x] **Task 5.5: Commit Phase 5 + the README**
```bash
git add scripts/twitter_threads/README.md
git commit -m "docs(twitter_threads): README — standalone usage + copy-to-another-repo instructions"
```
- [ ] **Task 5.6: Conductor — User Manual Verification (Protocol in workflow.md)**
- [~] **Task 5.6: Conductor — User Manual Verification (Protocol in workflow.md)** — handed off via TRACK_COMPLETION report (autonomous mode)
Present the verification results to the user. PAUSE for user confirmation before marking the track complete.
- [ ] **Task 5.7: Mark the track complete + write the TRACK_COMPLETION report**
- [x] **Task 5.7: Mark the track complete + write the TRACK_COMPLETION report**
WHERE: `docs/reports/TRACK_COMPLETION_twitter_threads_extraction_20260705.md`
WHAT: A short report (per the `tier2_autonomous_sandbox_20260616` precedent): what was done, files created, tests passing, the standalone-requirement verification, and any deferred items.
conductor/tracks/twitter_threads_extraction_20260705/state.toml
+43 -42
View File
@@ -1,12 +1,13 @@
# Track state for twitter_threads_extraction_20260705
# Initialized by Tier 1 Orchestrator on 2026-07-05.
# Completed by Tier 2 autonomous run on 2026-07-05.
# Standalone tooling track: scripts/twitter_threads/ + tests + README. No src/ changes.
[meta]
track_id = "twitter_threads_extraction_20260705"
name = "Twitter/X Thread Extraction Tooling"
status = "active"
current_phase = 0
status = "completed"
current_phase = 5
last_updated = "2026-07-05"
[blocked_by]
@@ -16,46 +17,46 @@ last_updated = "2026-07-05"
# None. The tooling may be consumed by a future "Twitter thread analysis campaign" track.
[phases]
phase_1 = { status = "pending", checkpointsha = "", name = "Scaffold + Error Types + Data Classes" }
phase_2 = { status = "pending", checkpointsha = "", name = "render_markdown.py (TDD-first; pure function)" }
phase_3 = { status = "pending", checkpointsha = "", name = "download_media.py (stdlib urllib + idempotent)" }
phase_4 = { status = "pending", checkpointsha = "", name = "fetch_thread.py (gallery-dl subprocess + html.parser fallback)" }
phase_5 = { status = "pending", checkpointsha = "", name = "README + End-to-End CLI + Verification" }
phase_1 = { status = "completed", checkpointsha = "161d8da8", name = "Scaffold + Error Types + Data Classes" }
phase_2 = { status = "completed", checkpointsha = "ef98f6d1", name = "render_markdown.py (TDD-first; pure function)" }
phase_3 = { status = "completed", checkpointsha = "f503eb5d", name = "download_media.py (stdlib urllib + idempotent)" }
phase_4 = { status = "completed", checkpointsha = "06ff9299", name = "fetch_thread.py (gallery-dl subprocess + html.parser fallback)" }
phase_5 = { status = "completed", checkpointsha = "fe207c1e", name = "README + End-to-End CLI + Verification" }
[tasks]
t1_1 = { status = "pending", commit_sha = "", description = "Create scripts/twitter_threads/ + __init__.py (namespace docstring)" }
t1_2 = { status = "pending", commit_sha = "", description = "Write error_types.py (ErrorInfo + make_error; standalone, no import from scripts.video_analysis)" }
t1_3 = { status = "pending", commit_sha = "", description = "Write typed dataclasses (PostData, PostMetrics, ThreadData; frozen, slots)" }
t1_4 = { status = "pending", commit_sha = "", description = "Write tests/test_twitter_threads_types.py (TDD red-first)" }
t1_5 = { status = "pending", commit_sha = "", description = "Commit Phase 1" }
t2_1 = { status = "pending", commit_sha = "", description = "Write failing tests for render_markdown.py (5+ tests: single post, thread, quote-tweet, media links, metrics, title)" }
t2_2 = { status = "pending", commit_sha = "", description = "Implement render_markdown.py (YAML front-matter + per-post sections + media links + quote-tweet blockquote)" }
t2_3 = { status = "pending", commit_sha = "", description = "Commit Phase 2" }
t3_1 = { status = "pending", commit_sha = "", description = "Write failing tests for download_media.py (4+ tests: naming, video naming, idempotent, http error)" }
t3_2 = { status = "pending", commit_sha = "", description = "Implement download_media.py (stdlib urllib; idempotent; typed naming <post_id>_<kind><index>.<ext>)" }
t3_3 = { status = "pending", commit_sha = "", description = "Commit Phase 3" }
t4_1 = { status = "pending", commit_sha = "", description = "Write failing tests for fetch_thread.py Strategy C (local HTML parse; 4+ tests)" }
t4_2 = { status = "pending", commit_sha = "", description = "Implement fetch_thread_from_html (html.parser.HTMLParser subclass; stdlib only)" }
t4_3 = { status = "pending", commit_sha = "", description = "Implement fetch_thread_from_url (gallery-dl subprocess wrapper; --dump-json + --write-metadata)" }
t4_4 = { status = "pending", commit_sha = "", description = "Implement CLI dispatch + --help (argparse; URL or local HTML path input)" }
t4_5 = { status = "pending", commit_sha = "", description = "Commit Phase 4" }
t5_1 = { status = "pending", commit_sha = "", description = "Write README.md (prerequisites, usage, output layout, copy-to-another-repo, Strategy B/D alternatives)" }
t5_2 = { status = "pending", commit_sha = "", description = "Verify standalone requirement (VC6 + VC7: --help works; zero src/conductor/video_analysis imports)" }
t5_3 = { status = "pending", commit_sha = "", description = "End-to-end smoke test (manual; user provides a real X.com URL + cookies.txt)" }
t5_4 = { status = "pending", commit_sha = "", description = "Run the full test suite for the new tests (batched verification)" }
t5_5 = { status = "pending", commit_sha = "", description = "Commit Phase 5 + README" }
t5_6 = { status = "pending", commit_sha = "", description = "Conductor — User Manual Verification (PAUSE for user confirmation)" }
t5_7 = { status = "pending", commit_sha = "", description = "Mark track complete + write TRACK_COMPLETION_twitter_threads_extraction_20260705.md" }
t1_1 = { status = "completed", commit_sha = "161d8da8", description = "Create scripts/twitter_threads/ + __init__.py (namespace docstring)" }
t1_2 = { status = "completed", commit_sha = "161d8da8", description = "Write error_types.py (ErrorInfo + make_error; standalone, no import from scripts.video_analysis)" }
t1_3 = { status = "completed", commit_sha = "161d8da8", description = "Write typed dataclasses (PostData, PostMetrics, ThreadData; frozen, slots)" }
t1_4 = { status = "completed", commit_sha = "161d8da8", description = "Write tests/test_twitter_threads_types.py (TDD red-first)" }
t1_5 = { status = "completed", commit_sha = "161d8da8", description = "Commit Phase 1" }
t2_1 = { status = "completed", commit_sha = "ef98f6d1", description = "Write failing tests for render_markdown.py (7 tests incl Result contract)" }
t2_2 = { status = "completed", commit_sha = "ef98f6d1", description = "Implement render_markdown.py (YAML front-matter + per-post sections + media links + quote-tweet blockquote)" }
t2_3 = { status = "completed", commit_sha = "ef98f6d1", description = "Commit Phase 2" }
t3_1 = { status = "completed", commit_sha = "f503eb5d", description = "Write failing tests for download_media.py (6 tests: naming, video naming, idempotent, http error)" }
t3_2 = { status = "completed", commit_sha = "f503eb5d", description = "Implement download_media.py (stdlib urllib; idempotent; typed naming <post_id>_<kind><index>.<ext>)" }
t3_3 = { status = "completed", commit_sha = "f503eb5d", description = "Commit Phase 3" }
t4_1 = { status = "completed", commit_sha = "06ff9299", description = "Write failing tests for fetch_thread.py Strategy C (local HTML parse; 7 tests incl URL normalization)" }
t4_2 = { status = "completed", commit_sha = "06ff9299", description = "Implement fetch_thread_from_html (html.parser.HTMLParser subclass; stdlib only)" }
t4_3 = { status = "completed", commit_sha = "06ff9299", description = "Implement fetch_thread_from_url (gallery-dl subprocess wrapper; --dump-json; best-effort, network-boundary)" }
t4_4 = { status = "completed", commit_sha = "06ff9299", description = "Implement CLI dispatch + --help (argparse; URL or local HTML path input); _normalize_url strips ?s=20" }
t4_5 = { status = "completed", commit_sha = "06ff9299", description = "Commit Phase 4" }
t5_1 = { status = "completed", commit_sha = "fe207c1e", description = "Write README.md (prerequisites, usage, output layout, copy-to-another-repo, Strategy B/D alternatives)" }
t5_2 = { status = "completed", commit_sha = "fe207c1e", description = "Verify standalone requirement (VC6 + VC7: --help works; zero src/conductor/video_analysis imports)" }
t5_3 = { status = "completed", commit_sha = "", description = "End-to-end extraction: 8 URLs -> 4 merged threads (conversation dedupe) in docs/twitter/ with gallery-dl media (images embedded, video as <video>). Driver: conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py" }
t5_4 = { status = "completed", commit_sha = "ed867317", description = "Run the full new-test suite (34 tests pass together; log at tests/artifacts/tier2_state/.../test_run_phase5_final.log)" }
t5_5 = { status = "completed", commit_sha = "fe207c1e", description = "Commit Phase 5 + README" }
t5_6 = { status = "pending_user", commit_sha = "", description = "User Manual Verification — handed off via TRACK_COMPLETION report (autonomous mode: user reviews report to decide merge)" }
t5_7 = { status = "completed", commit_sha = "", description = "Mark track complete + write TRACK_COMPLETION_twitter_threads_extraction_20260705.md" }
[verification]
vc1_package_exists = false
vc2_readme_exists = false
vc3_render_tests_pass = false
vc4_fetch_tests_pass = false
vc5_media_tests_pass = false
vc6_standalone_help_works = false
vc7_zero_internal_imports = false
vc8_corpus_extraction_complete = false
vc1_package_exists = true
vc2_readme_exists = true
vc3_render_tests_pass = true
vc4_fetch_tests_pass = true
vc5_media_tests_pass = true
vc6_standalone_help_works = true
vc7_zero_internal_imports = true
vc8_corpus_extraction_complete = true # done: 8 URLs -> 4 merged threads in docs/twitter/ (deduped, media embedded), committed by user
[reference_project]
path = "C:\\projects\\forth\\bootslop"
@@ -75,10 +76,10 @@ acceptance_criterion = "All 8 threads extract cleanly to thread.md + media/ pair
[standalone_requirement]
rationale = "Per user directive: scripts must be standalone tooling so they don't have lots of dependencies with the repo's codebase and could be utilized in others."
enforcement = "VC7 — zero imports from src/, conductor/, or scripts.video_analysis. Verified via grep."
copyable = "The scripts/twitter_threads/ directory is copy-pasteable to another repo with only gallery-dl as an external dep."
enforcement = "VC7 — zero imports from src/, conductor/, or scripts.video_analysis. Verified via grep (empty)."
copyable = "The scripts/twitter_threads/ directory is copy-pasteable to another repo with only gallery-dl as an external dep. All runnable modules use a dual-import (absolute/relative) so they work via -m and as bare scripts."
[acquisition_strategy]
primary = "Strategy A: gallery-dl subprocess wrapper (URL input; --dump-json + --write-metadata)"
primary = "Strategy A: gallery-dl subprocess wrapper (URL input; --dump-json)"
fallback = "Strategy C: local HTML parse via html.parser.HTMLParser (user saves HTML via browser; script parses offline)"
documented_alternatives = "Strategy B: snscrape (legacy; documented in README); Strategy D: tweepy/twitter-api-python (paid API; documented in README)"
documented_alternatives = "Strategy B: snscrape (legacy; documented in README); Strategy D: tweepy/twitter-api-python (paid API; documented in README)"