ed/manual_slop
Private
Public Access
0
0
Fork 0
Code Activity

conductor(track): init twitter_threads_extraction_20260705 — standalone Twitter/X thread extraction tooling

Browse Source 
Scripts + workflow for extracting Twitter/X posts and threads into
Markdown with associated media. Mirrors the scripts/video_analysis/
pattern. Standalone requirement: zero imports from src/, conductor/,
or scripts.video_analysis — copy-pasteable to another repo with only
gallery-dl as the external dep.

5 modules: __init__.py, error_types.py (Result[T, ErrorInfo] +
ThreadData/PostData typed dataclasses), fetch_thread.py (gallery-dl
subprocess for URLs + html.parser fallback for local HTML),
download_media.py (stdlib urllib, idempotent), render_markdown.py
(YAML front-matter + per-post sections + ./media/ links).

Reference project: C:\projects\forth\bootslop — the corpus feeds
bootslop's scripts and reference-generation pipeline. Acceptance
corpus: 8 threads (@NOTimothyLottes x6 + @VPCOMPRESSB x2) extracted
to tests/artifacts/twitter_threads_corpus/. The ?s=20 quote-share
suffix on the @VPCOMPRESSB URLs must be stripped by fetch_thread.py
before acquisition (added to FR2 as URL normalization).

5 phases / 23 tasks. 8 verification criteria (VC1-VC8). TDD red-first
on the pure-function modules (render_markdown, types, media naming).
This commit is contained in:
Edward R. Gonzalez
2026-07-05 16:50:05 -04:00
parent 4c9fc99cd4
commit 0908f8fa28
6 changed files with 681 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conductor/chronology.md
+1
View File
@@ -2,6 +2,7 @@
| Date | ID | Status | Summary | Folder | Range |
| --- | --- | --- | --- | --- | --- |
| 2026-07-05 | `twitter_threads_extraction_20260705` | Active | Standalone Python scripts + workflow for extracting Twitter/X posts and threads into Markdown with associated media. Mirrors the `scripts/video_analysis/` pattern. spec ✓, plan ✓, ready to start. | `conductor/tracks/twitter_threads_extraction_20260705` | (not yet started) |
| 2026-07-05 | `superpowers_review_apply_high_20260705` | Completed | Phase 1: `26dd9258` Session Start Checklist item 13 added; Phase 2: `670a919e` tests/test_mma_skill_discipline.py (25 cases); Phase 3: `5037f48f` cross-reference + finalization. | `conductor/archive/superpowers_review_apply_high_20260705` | `0522252f..5037f48f` (3) |
| 2026-07-05 | `superpowers_review_20260619` | Completed | 16-section review of the superpowers skill suite: 14 PARITY / 3 PARTIAL / 2 ARCH-DIFF / 1 GAP; 25 decisions logged; 3 side artifacts (decisions.md, comparison_table.md, nagent_takeaways). | `conductor/archive/superpowers_review_20260619` | `eb272cb7..137868a1` (40) |
| 2026-07-05 | `agent_directives_consolidation_20260705` | Completed | AGENTS.md reduced from 202→87 lines; conductor/workflow.md §Process Anti-Patterns promoted to canonical home; conductor/edit_workflow.md §9 + product-guidelines.md subsections thin-pointed to canonical styleguides; 9 atomic commits with git notes. | `conductor/archive/agent_directives_consolidation_20260705` | `48ddd15c..8ab82022` (4) |
conductor/tracks.md
+2
View File
@@ -21,6 +21,7 @@ Tracks that are unblocked and ready to start. Ordered by **dependency** (blocked
| 4 | A | [MCP Architecture Refactor (Sub-MCP Extraction)](#track-mcp-architecture-refactor) | spec ✓, plan pending | (none) |
| 22b | A | [Meta-Tooling Workflow Review](#track-meta-tooling-workflow-review) | spec ✓, plan ✓, parked | (none) |
| 23 | A | [Intent-Based Scripting Languages Survey](#track-intent-dsl-survey) | spec ✓, plan pending | (none) |
| 30 | A | [Twitter/X Thread Extraction Tooling](#track-twitter-threads-extraction) | spec ✓, plan ✓, ready to start | (none) |
| 7 | — | [UI Polish (Five Issues)](#track-ui-polish-five-issues) | spec ✓, plan ✓, ready to start | (none) |
| 19 | — | [Context First Message Fix](#track-context-first-message-fix) | spec TBD | (none) |
@@ -30,6 +31,7 @@ Tracks that are unblocked and ready to start. Ordered by **dependency** (blocked
- **MCP Architecture Refactor** → `conductor/tracks/mcp_architecture_refactor_20260606/`
- **Meta-Tooling Workflow Review** → `conductor/tracks/meta_tooling_workflow_review_20260620/`
- **Intent-Based Scripting Languages Survey** → `conductor/tracks/intent_dsl_survey_20260612/`
- **Twitter/X Thread Extraction Tooling** → `conductor/tracks/twitter_threads_extraction_20260705/`
- **UI Polish (Five Issues)** → `conductor/tracks/` (see spec for details)
- **Context First Message Fix** → `conductor/tracks/context_first_message_fix_20260604/`
conductor/tracks/twitter_threads_extraction_20260705/metadata.json
+91
View File
@@ -0,0 +1,91 @@
{
"track_id": "twitter_threads_extraction_20260705",
"name": "Twitter/X Thread Extraction Tooling",
"status": "active",
"branch": "master",
"created": "2026-07-05",
"owner": "Tier 1 (initialized); implementation delegated to Tier 2/3.",
"blocked_by": [],
"blocks": [],
"scope": {
"new_files": [
"scripts/twitter_threads/__init__.py",
"scripts/twitter_threads/error_types.py",
"scripts/twitter_threads/fetch_thread.py",
"scripts/twitter_threads/download_media.py",
"scripts/twitter_threads/render_markdown.py",
"scripts/twitter_threads/README.md",
"tests/test_twitter_threads_types.py",
"tests/test_twitter_threads_render.py",
"tests/test_twitter_threads_media.py",
"tests/test_twitter_threads_fetch.py",
"docs/reports/TRACK_COMPLETION_twitter_threads_extraction_20260705.md"
],
"modified_files": [],
"deleted_files": []
},
"estimated_effort": {
"method": "scope (per workflow.md Tier 1 Track Initialization Rules. NO day estimates.)",
"phase_1": "5 tasks: scaffold + error types + dataclasses + tests + commit",
"phase_2": "3 tasks: failing tests + render_markdown impl + commit",
"phase_3": "3 tasks: failing tests + download_media impl + commit",
"phase_4": "5 tasks: failing tests + HTML parse + URL path + CLI + commit",
"phase_5": "7 tasks: README + standalone verification + smoke test + batch tests + commit + user verification + completion report"
},
"verification_criteria": [
"VC1: scripts/twitter_threads/ exists with 5 files: __init__.py, error_types.py, fetch_thread.py, download_media.py, render_markdown.py",
"VC2: scripts/twitter_threads/README.md exists and documents prerequisites, usage, output layout, and the copy-to-another-repo instructions",
"VC3: tests/test_twitter_threads_render.py passes — given a ThreadData fixture, render_markdown.py produces the expected Markdown (YAML front-matter + per-post sections + media links)",
"VC4: tests/test_twitter_threads_fetch.py passes — given a small fixture HTML file, fetch_thread.py (Strategy C path) produces the expected ThreadData",
"VC5: tests/test_twitter_threads_media.py passes — download_media.py produces correctly-named files and skips existing files",
"VC6: python scripts/twitter_threads/fetch_thread.py --help works (standalone invocation, no src/ imports)",
"VC7: zero imports from src/, conductor/, or scripts.video_analysis in scripts/twitter_threads/ (verified via grep)",
"VC8: end-to-end smoke test against the 8-thread corpus (@NOTimothyLottes + @VPCOMPRESSB) produces 8 thread.md + media/ pairs in tests/artifacts/twitter_threads_corpus/ — the acceptance corpus for the C:\\projects\\forth\\bootslop reference-generation pipeline"
],
"regressions_and_pre_existing_failures": [],
"pre_existing_failures_remaining": [],
"deferred_to_followup_tracks": [
{
"title": "Twitter thread analysis campaign (the analog of the video analysis campaign)",
"description": "If the user wants to analyze many threads + cross-thread synthesis, that's a separate future track that consumes this tooling. This track ships the tooling only.",
"track_status": "not yet initialized; depends on user demand"
},
{
"title": "RAG indexing of extracted threads",
"description": "The output Markdown could be RAG-indexed for retrieval. Out of scope for this track.",
"track_status": "future; depends on user demand"
},
{
"title": "Twitter API v2 paid-endpoint backend (Strategy D)",
"description": "Documented in the README as an alternative acquisition backend; not implemented as the default.",
"track_status": "future; depends on user having developer credentials"
}
],
"risk_register": [
{
"id": "R1",
"description": "gallery-dl breaks against X.com (auth wall, rate limit, or X.com API change)",
"likelihood": "medium",
"impact": "The URL acquisition path (Strategy A) fails; the user must fall back to the local-HTML-parse path (Strategy C)",
"mitigation": "Strategy C is implemented as the fallback; the README documents the manual-export-then-parse workflow"
},
{
"id": "R2",
"description": "X.com media CDN URLs require auth (the public-CDN assumption is wrong)",
"likelihood": "low",
"impact": "download_media.py fails to fetch media even after fetch_thread.py succeeds",
"mitigation": "the README documents the cookies.txt workaround; download_media.py can be extended to pass cookies via urllib"
},
{
"id": "R3",
"description": "The local-HTML-parse path (Strategy C) is brittle against X.com's HTML structure changing",
"likelihood": "medium",
"impact": "the offline fallback stops working",
"mitigation": "the parser is stdlib-only (html.parser) and tested against fixture HTML; if X.com changes structure, the user re-exports HTML and the parser is updated"
}
],
"standalone_requirement": {
"rationale": "Per user directive: scripts must be standalone tooling so they don't have lots of dependencies with the repo's codebase and could be utilized in others.",
"enforcement": "VC7 — zero imports from src/, conductor/, or scripts.video_analysis. Verified via grep. The scripts/twitter_threads/ directory is copy-pasteable to another repo with only gallery-dl as an external dep."
}
}
conductor/tracks/twitter_threads_extraction_20260705/plan.md
+292
View File
@@ -0,0 +1,292 @@
# Plan: Twitter/X Thread Extraction Tooling
Track: `twitter_threads_extraction_20260705`
Branch: master (scripts + tests only; no `src/` changes, no GUI changes)
Spec: `conductor/tracks/twitter_threads_extraction_20260705/spec.md`
Standalone tooling track. The deliverable is a `scripts/twitter_threads/` package + tests + README. No application code changes.
---
## Phase 1: Scaffold + Error Types + Data Classes
Focus: create the package directory, the shared error type, and the typed dataclasses (`ThreadData`, `PostData`) that the rest of the pipeline passes around.
- [ ] **Task 1.1: Create the `scripts/twitter_threads/` directory + `__init__.py`**
WHERE: `scripts/twitter_threads/__init__.py`
WHAT: A docstring documenting the namespace, the per-module responsibilities, and the standalone-usage note (no `src/` imports; copy-pasteable to another repo).
HOW: Mirror `scripts/video_analysis/__init__.py`.
- [ ] **Task 1.2: Write `error_types.py` (the shared Result[T, ErrorInfo] shape)**
WHERE: `scripts/twitter_threads/error_types.py`
WHAT: Copy the shape of `scripts/video_analysis/error_types.py`: `ErrorInfo` dataclass (frozen, slots) + `make_error` factory. Standalone — no import from `scripts.video_analysis`.
HOW:
```python
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True, slots=True)
class ErrorInfo:
kind: str
source: str
detail: str
def make_error(kind: str, source: str, detail: str) -> ErrorInfo:
return ErrorInfo(kind=kind, source=source, detail=detail)
```
VERIFY: `python -c "from scripts.twitter_threads.error_types import ErrorInfo, make_error; print(make_error('X','y','z'))"` prints the dataclass repr.
- [ ] **Task 1.3: Write the typed dataclasses (`ThreadData`, `PostData`)**
WHERE: `scripts/twitter_threads/error_types.py` (same file; keeps the standalone file count low) OR a new `scripts/twitter_threads/types.py` (if the user prefers separation). Default: same file.
WHAT:
```python
@dataclass(frozen=True, slots=True)
class PostData:
post_id: str
author: str
handle: str
text: str
timestamp: str # ISO 8601
media_urls: tuple[str, ...]
reply_to_id: str | None
quote_of_id: str | None
metrics: PostMetrics
@dataclass(frozen=True, slots=True)
class PostMetrics:
reply_count: int
repost_count: int
like_count: int
view_count: int | None
@dataclass(frozen=True, slots=True)
class ThreadData:
root_post_id: str
posts: tuple[PostData, ...]
source_url: str
```
HOW: Per `conductor/code_styleguides/data_oriented_design.md` §8.5 — typed, frozen, slots. No `dict[str, Any]`.
VERIFY: `python -c "from scripts.twitter_threads.error_types import PostData; print(PostData.__slots__)"` prints the slots tuple.
- [ ] **Task 1.4: Write tests for the error types + dataclasses**
WHERE: `tests/test_twitter_threads_types.py`
WHAT: Construct `ErrorInfo`, `PostData`, `ThreadData` instances. Assert field access, frozen-ness (mutation raises `FrozenInstanceError`), slots-ness (no `__dict__`).
HOW: Per the TDD red-first protocol — write the tests first, run them (they fail because the types don't exist yet), then implement Task 1.2 + 1.3 to make them pass.
VERIFY: `uv run pytest tests/test_twitter_threads_types.py -v` passes.
- [ ] **Task 1.5: Commit Phase 1**
```bash
git add scripts/twitter_threads/__init__.py scripts/twitter_threads/error_types.py tests/test_twitter_threads_types.py
git commit -m "feat(twitter_threads): scaffold package + error types + typed dataclasses"
```
---
## Phase 2: `render_markdown.py` (the pure-function module; TDD-first)
Focus: the Markdown emission module. This is pure (no network, no subprocess), so it's the easiest to TDD and the highest-confidence module.
- [ ] **Task 2.1: Write failing tests for `render_markdown.py`**
WHERE: `tests/test_twitter_threads_render.py`
WHAT: 5+ tests:
1. `test_render_single_post` — a `ThreadData` with 1 post; assert the YAML front-matter has the right fields; assert the post body is rendered; assert media links are `./media/<filename>`.
2. `test_render_thread` — a `ThreadData` with 3 posts (a thread); assert 3 `## Post N` sections in chronological order; assert each has the timestamp + reply-to marker.
3. `test_render_quote_tweet` — a `ThreadData` with a post that has `quote_of_id` set; assert the quote-tweet renders as a nested blockquote with a link.
4. `test_render_media_links` — a post with 2 image URLs + 1 video URL; assert 3 media links, named `<post_id>_img1.jpg`, `<post_id>_img2.jpg`, `<post_id>_vid1.mp4`.
5. `test_render_metrics_in_frontmatter` — assert `reply_count`, `repost_count`, `like_count`, `view_count` appear in the YAML front-matter; assert `view_count: null` when `view_count` is `None`.
6. `test_render_title_from_first_post` — the thread title is the first post's first line (truncated to 80 chars).
HOW: Construct `ThreadData` fixtures inline (no network). Assert on the returned Markdown string.
VERIFY: `uv run pytest tests/test_twitter_threads_render.py` FAILS (module doesn't exist yet).
- [ ] **Task 2.2: Implement `render_markdown.py`**
WHERE: `scripts/twitter_threads/render_markdown.py`
WHAT: The `render_markdown(thread: ThreadData, media_paths: dict[str, list[Path]], output: Path) -> Result[Path, ErrorInfo]` function.
HOW:
- Build the YAML front-matter from `thread.posts[0]` (the root post's metadata).
- Build the body: iterate `thread.posts` in order; for each, emit `## Post N (<timestamp>)` + optional `— reply to Post M` marker.
- For each `media_url` in `post.media_urls`: emit `[Media K](./media/<post_id>_<kind><K>.<ext>)`.
- For quote-tweets: emit a `> [Quoting @<handle>](<quoted_url>)` blockquote.
- Write to `output` via `Path.write_text(content, encoding="utf-8")`.
- Return `Result.ok(output)` on success; `Result.err(...)` on file-write failure.
VERIFY: `uv run pytest tests/test_twitter_threads_render.py` PASSES.
- [ ] **Task 2.3: Commit Phase 2**
```bash
git add scripts/twitter_threads/render_markdown.py tests/test_twitter_threads_render.py
git commit -m "feat(twitter_threads): render_markdown.py — YAML front-matter + per-post sections + media links"
```
---
## Phase 3: `download_media.py` (the media downloader)
Focus: download the media files referenced by a `ThreadData`. Mock the HTTP in tests; real network only at runtime.
- [ ] **Task 3.1: Write failing tests for `download_media.py`**
WHERE: `tests/test_twitter_threads_media.py`
WHAT: 4+ tests:
1. `test_download_naming` — a post with 2 image URLs (`...?format=jpg&name=large`, `...?format=png&name=large`); assert the downloaded files are named `<post_id>_img1.jpg` and `<post_id>_img2.png`.
2. `test_download_video_naming` — a post with a video URL (`.../vid/.../1234567890.mp4`); assert the file is named `<post_id>_vid1.mp4`.
3. `test_download_idempotent` — pre-create the target file with the expected byte size; assert the download is skipped (no HTTP call made).
4. `test_download_http_error` — mock `urlopen` to raise `URLError`; assert `Result.err` with `ErrorInfo.kind == "HttpError"`.
HOW: Mock `urllib.request.urlopen` via `unittest.mock.patch` (this is a boundary test — the HTTP boundary — so mocking is allowed per the structural testing contract). Use `tests/artifacts/twitter_threads_media_<test_name>/` as the output dir per the workspace-paths convention.
VERIFY: `uv run pytest tests/test_twitter_threads_media.py` FAILS.
- [ ] **Task 3.2: Implement `download_media.py`**
WHERE: `scripts/twitter_threads/download_media.py`
WHAT: The `download_media(thread: ThreadData, output_dir: Path) -> Result[list[Path], ErrorInfo]` function.
HOW:
- For each post in `thread.posts`, for each `media_url` in `post.media_urls`:
- Derive the kind (`img` / `vid` / `gif`) from the URL or Content-Type.
- Derive the extension (`.jpg`, `.png`, `.mp4`).
- Compute the target filename: `<post_id>_<kind><index>.<ext>`.
- If the target file exists and has byte size > 0: skip (idempotent).
- Else: `urllib.request.urlopen(media_url)` → read bytes → write to target.
- Return `Result.ok(list_of_paths)` on success; `Result.err(ErrorInfo)` on the first HTTP failure.
- Use only stdlib (`urllib.request`, `pathlib`, `dataclasses`). No `requests`.
VERIFY: `uv run pytest tests/test_twitter_threads_media.py` PASSES.
- [ ] **Task 3.3: Commit Phase 3**
```bash
git add scripts/twitter_threads/download_media.py tests/test_twitter_threads_media.py
git commit -m "feat(twitter_threads): download_media.py — stdlib urllib + idempotent + typed naming"
```
---
## Phase 4: `fetch_thread.py` (the acquisition module; the network boundary)
Focus: acquire a `ThreadData` from either a URL (via `gallery-dl` subprocess) or a local HTML file (Strategy C parse). The URL path is the network boundary; the HTML path is the offline fallback.
- [ ] **Task 4.1: Write failing tests for the local-HTML-parse path (Strategy C)**
WHERE: `tests/test_twitter_threads_fetch.py`
WHAT: 3+ tests:
1. `test_parse_html_single_post` — a small fixture HTML file (in `tests/artifacts/`) containing one tweet's text + author + timestamp + 1 image URL; assert the parsed `ThreadData` has 1 post with the right fields.
2. `test_parse_html_thread` — a fixture HTML file with a 3-post thread; assert 3 `PostData` instances in chronological order.
3. `test_parse_html_no_media` — a post with no media; assert `media_urls == ()`.
4. `test_parse_html_malformed` — a malformed HTML file; assert `Result.err` with `ErrorInfo.kind == "ParseError"`.
HOW: Use `html.parser.HTMLParser` (stdlib) — the test fixtures are real HTML strings. No network.
VERIFY: `uv run pytest tests/test_twitter_threads_fetch.py` FAILS.
- [ ] **Task 4.2: Implement the local-HTML-parse path**
WHERE: `scripts/twitter_threads/fetch_thread.py`
WHAT: A `fetch_thread_from_html(html_path: Path) -> Result[ThreadData, ErrorInfo]` function that parses a local HTML file using `html.parser.HTMLParser` (stdlib, no BeautifulSoup).
HOW:
- Subclass `HTMLParser`; walk the DOM; extract post text (the tweet-text container), author, handle, timestamp, media URLs (the `img[src]` and `video[src]` tags in the tweet container).
- Construct `PostData` instances; chain them into a `ThreadData`.
- Return `Result.err(ErrorInfo("ParseError", "fetch_thread_from_html", detail))` on malformed HTML.
VERIFY: `uv run pytest tests/test_twitter_threads_fetch.py` PASSES for the Strategy C tests.
- [ ] **Task 4.3: Implement the URL path (the `gallery-dl` subprocess wrapper)**
WHERE: `scripts/twitter_threads/fetch_thread.py` (same file; add the URL path)
WHAT: A `fetch_thread_from_url(url: str, cookies_path: Path | None = None) -> Result[ThreadData, ErrorInfo]` function.
HOW:
- Build the `gallery-dl` args: `["gallery-dl", "--dump-json", "--write-metadata", url]`. If `cookies_path` is provided, add `--cookies <cookies_path>`.
- `subprocess.run(args, capture_output=True, text=True)`.
- Parse the JSON stdout: `gallery-dl` emits one JSON object per post. Parse each into a `PostData`.
- Chain into `ThreadData`.
- On `gallery-dl` non-zero exit: return `Result.err(ErrorInfo("GalleryDlError", "fetch_thread_from_url", stderr[:500]))`.
- On JSON parse failure: return `Result.err(ErrorInfo("JsonParseError", ...))`.
VERIFY: Manual smoke test against a real X.com URL (the user provides a URL + cookies file). The automated tests cover the HTML path only (Strategy C); the URL path is the network boundary and is smoke-tested manually.
- [ ] **Task 4.4: Implement the CLI dispatch + `--help`**
WHERE: `scripts/twitter_threads/fetch_thread.py` (the `if __name__ == "__main__":` block)
WHAT: A CLI that takes a URL or a local HTML path + `--output` + optional `--cookies`, dispatches to the right function, and writes the `ThreadData` to a JSON file (intermediate) for downstream `download_media.py` + `render_markdown.py` to consume.
HOW:
```python
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="Fetch a Twitter/X thread into a ThreadData JSON.")
parser.add_argument("source", help="X.com URL or local HTML file path")
parser.add_argument("--output", type=Path, required=True, help="Output directory")
parser.add_argument("--cookies", type=Path, default=None, help="cookies.txt for gallery-dl auth")
args = parser.parse_args()
...
```
VERIFY: `python scripts/twitter_threads/fetch_thread.py --help` works (standalone, no `src/` imports).
- [ ] **Task 4.5: Commit Phase 4**
```bash
git add scripts/twitter_threads/fetch_thread.py tests/test_twitter_threads_fetch.py
git commit -m "feat(twitter_threads): fetch_thread.py — gallery-dl subprocess (URL) + html.parser (local HTML fallback)"
```
---
## Phase 5: README + End-to-End CLI + Verification
Focus: the standalone-usage README, an end-to-end CLI test, and the final verification.
- [ ] **Task 5.1: Write `README.md`**
WHERE: `scripts/twitter_threads/README.md`
WHAT: Per spec FR7 — prerequisites (`gallery-dl`, `cookies.txt`, Python 3.11+), usage (both `uv run` and standalone `python`), output layout, "copy to another repo" instructions, and the Strategy B/D documentation as alternatives.
HOW: Markdown; no code.
- [ ] **Task 5.2: Verify the standalone requirement (VC6 + VC7)**
WHAT:
- `python scripts/twitter_threads/fetch_thread.py --help` works from the repo root with no `uv run` (standalone invocation).
- `grep -r "from src\." scripts/twitter_threads/` returns nothing.
- `grep -r "import src\." scripts/twitter_threads/` returns nothing.
- `grep -r "from conductor\." scripts/twitter_threads/` returns nothing.
- `grep -r "from scripts\.video_analysis" scripts/twitter_threads/` returns nothing.
VERIFY: All greps return empty; the `--help` works.
- [ ] **Task 5.3: End-to-end smoke test against the 8-thread corpus (@NOTimothyLottes + @VPCOMPRESSB)**
WHAT: Run the full pipeline against all 8 reference URLs from the spec's "Reference Project + Test Corpus" section. These are the acceptance corpus for the `C:\projects\forth\bootslop` reference-generation pipeline.
URLs:
1. `https://x.com/NOTimothyLottes/status/1757198624818168210`
2. `https://x.com/NOTimothyLottes/status/1653570742762479620`
3. `https://x.com/NOTimothyLottes/status/1917646466417381426`
4. `https://x.com/NOTimothyLottes/status/1917645859791200562`
5. `https://x.com/NOTimothyLottes/status/1917644904055910502`
6. `https://x.com/NOTimothyLottes/status/1917642786804785230`
7. `https://x.com/VPCOMPRESSB/status/1991383117571957052` (note: `?s=20` query suffix must be stripped by `fetch_thread.py` before acquisition)
8. `https://x.com/VPCOMPRESSB/status/1987744335333622188` (note: `?s=20` query suffix must be stripped)
HOW (per URL):
```bash
uv run python -m scripts.twitter_threads.fetch_thread "<url>" --output ./tests/artifacts/twitter_threads_corpus/ --cookies ./cookies.txt
uv run python -m scripts.twitter_threads.download_media --input ./tests/artifacts/twitter_threads_corpus/<id>/thread_data.json --output ./tests/artifacts/twitter_threads_corpus/<id>/media/
uv run python -m scripts.twitter_threads.render_markdown --input ./tests/artifacts/twitter_threads_corpus/<id>/thread_data.json --media-dir ./tests/artifacts/twitter_threads_corpus/<id>/media/ --output ./tests/artifacts/twitter_threads_corpus/<id>/thread.md
```
VERIFY: 8 `thread.md` files exist with YAML front-matter + post sections + media links; 8 `media/` directories have the downloaded assets. This is the acceptance corpus for the track — if all 8 extract cleanly, the track ships and the corpus is handed off to `bootslop`. This is a manual verification (network boundary; the user provides the cookies.txt).
- [ ] **Task 5.4: Run the full test suite for the new tests**
WHAT: `uv run pytest tests/test_twitter_threads_types.py tests/test_twitter_threads_render.py tests/test_twitter_threads_media.py tests/test_twitter_threads_fetch.py -v`
VERIFY: All tests pass. This is the batched verification (the only verification that matters per the Isolated-Pass Verification Fallacy rule).
- [ ] **Task 5.5: Commit Phase 5 + the README**
```bash
git add scripts/twitter_threads/README.md
git commit -m "docs(twitter_threads): README — standalone usage + copy-to-another-repo instructions"
```
- [ ] **Task 5.6: Conductor — User Manual Verification (Protocol in workflow.md)**
Present the verification results to the user. PAUSE for user confirmation before marking the track complete.
- [ ] **Task 5.7: Mark the track complete + write the TRACK_COMPLETION report**
WHERE: `docs/reports/TRACK_COMPLETION_twitter_threads_extraction_20260705.md`
WHAT: A short report (per the `tier2_autonomous_sandbox_20260616` precedent): what was done, files created, tests passing, the standalone-requirement verification, and any deferred items.
HOW: Commit + update `conductor/tracks/twitter_threads_extraction_20260705/state.toml` to `status = "completed"`.
conductor/tracks/twitter_threads_extraction_20260705/spec.md
+211
View File
@@ -0,0 +1,211 @@
# Track Specification: Twitter/X Thread Extraction Tooling
## Overview
Build standalone, reusable Python scripts + a workflow for extracting Twitter/X posts and threads into Markdown, with their associated media (images, videos, GIFs) downloaded and linked alongside the post text. The output should be a self-contained Markdown document per thread (or single post) plus a sibling `media/` directory holding the downloaded assets, so the resulting artifact can be archived, read offline, or fed into downstream analysis pipelines without any dependency on the live X.com service.
This is the Twitter/X analog of the existing `scripts/video_analysis/` campaign tooling: a small, dependency-light pipeline that acquires source content losslessly and emits Markdown. It is NOT integrated into the Manual Slop application runtime — it is standalone tooling that lives under `scripts/twitter_threads/` and can be copied to other repos with minimal deps.
**Why standalone:** the user explicitly requested these scripts "be standalone tooling so they don't have lots of dependencies with the repo's codebase and could be utilized in others." The scripts must not import from `src/` and must not depend on the application's config, paths, or runtime. The only shared dependency is the project's Python toolchain (uv + pyproject.toml extras) for optional convenience.
## Reference Project + Test Corpus (the @NOTimothyLottes threads)
**Reference project:** `C:\projects\forth\bootslop` — a sibling project the user is building. The Twitter thread extraction tooling produced by this track is intended to feed reference material into `bootslop`'s scripts and reference-generation pipeline. The 6 threads below (all from `@NOTimothyLottes`) are the initial test corpus the tooling must handle. They are the first batch the user wants extracted, and they double as the concrete acceptance test for the scripts: if the tooling can extract all 6 cleanly, the track ships.
The threads cover a mix of single posts and multi-post threads (the @NOTimothyLottes account frequently posts long technical threads on graphics, low-level systems, and performance), so they exercise both the single-post path and the thread-chain path of `fetch_thread.py`.
| # | URL | Notes |
|---|---|---|
| 1 | `https://x.com/NOTimothyLottes/status/1757198624818168210` | Earlier thread (2024-02); older X.com URL format. Tests the URL path against a pre-rename-era post. |
| 2 | `https://x.com/NOTimothyLottes/status/1653570742762479620` | Earliest in the set (2023-05); tests legacy content acquisition. |
| 3 | `https://x.com/NOTimothyLottes/status/1917646466417381426` | Recent (2025-04); tests the current X.com URL format + auth-wall behavior. |
| 4 | `https://x.com/NOTimothyLottes/status/1917645859791200562` | Recent; likely a sibling post in the same thread as #3 (sequential post IDs). Tests thread-chain detection. |
| 5 | `https://x.com/NOTimothyLottes/status/1917644904055910502` | Recent; likely a sibling in the same thread as #3-4. |
| 6 | `https://x.com/NOTimothyLottes/status/1917642786804785230` | Recent; likely the root of the #3-4-5 thread (lowest post ID in the cluster). Tests root-post detection + the thread-chain walk from root. |
| 7 | `https://x.com/VPCOMPRESSB/status/1991383117571957052` | Recent (2025); different author (`@VPCOMPRESSB`). Tests the tooling against a second account. The `?s=20` query param (tweet quote-share suffix) must be stripped before passing to `gallery-dl`. |
| 8 | `https://x.com/VPCOMPRESSB/status/1987744335333622188` | Recent; second author thread. Tests corpus extraction across multiple authors in a single batch. |
**Acceptance criterion (added):** the end-to-end smoke test (Plan Task 5.3) must run the pipeline against all 8 URLs and produce 8 `thread.md` + `media/` pairs in `tests/artifacts/twitter_threads_corpus/`. The corpus is the deliverable that gets handed off to `bootslop`. The `?s=20` query-share suffix on the `@VPCOMPRESSB` URLs must be stripped by `fetch_thread.py` before acquisition (the suffix is a quote-share context param, not part of the post identity).
## Current State Audit (as of master `4c9fc99c`)
### Already Implemented (DO NOT re-implement)
- **`scripts/video_analysis/`** — the precedent pipeline (7 modules: `download_video.py`, `extract_transcript.py`, `extract_keyframes.py`, `ocr_frames.py`, `synthesize_report.py`, `error_types.py`, `__init__.py`). This track mirrors that pattern: a `__init__.py` documenting the namespace, an `error_types.py` for shared Result[T, ErrorInfo], and per-stage modules.
- **`scripts/video_analysis/error_types.py`** — the canonical `ErrorInfo` + `make_error` shape. The Twitter scripts reuse the same shape (copied, not imported — standalone requirement).
- **`conductor/code_styleguides/error_handling.md`** — the Result[T] + NIL_T sentinel pattern. The Twitter scripts follow it for their internal error handling.
- **`conductor/code_styleguides/python.md`** §1 — 1-space indentation, type hints, no comments in implementation code.
### Gaps to Fill (This Track's Scope)
- **GAP-1: No Twitter/X extraction scripts exist.** There is no `scripts/twitter_threads/` directory. The user wants the equivalent of `scripts/video_analysis/` for Twitter/X content.
- **GAP-2: No defined workflow for acquiring thread + media.** Twitter/X has no public transcript API like YouTube does; acquisition requires either (a) authenticated API access via `tweepy`/`twitter-api-python` (requires developer credentials), (b) guest/anonymous scraping via `gallery-dl` or `snscrape` (increasingly broken since the X.com rename + auth walls), or (c) manual export via the user's browser followed by script-driven post-processing. The track must pick a strategy and document the tradeoffs.
- **GAP-3: No Markdown emission format.** The output Markdown schema (how a thread renders, how media is linked, how quote-tweets and replies are represented, how metadata is captured) is undefined. The track must define it.
- **GAP-4: No README/workflow doc for the standalone tooling.** Per the standalone requirement, the scripts must ship with a README explaining how to use them outside this repo.
## Goals
- **G1.** A `scripts/twitter_threads/` package with a small pipeline: `fetch_thread.py` (acquire post + thread + quote-tweets), `download_media.py` (download images/videos/GIFs referenced by the posts), `render_markdown.py` (emit the Markdown document + media links), `error_types.py` (shared Result[T, ErrorInfo]), `__init__.py` (namespace docstring).
- **G2.** A defined Markdown output schema: one `.md` file per thread (or single post), with a YAML front-matter block capturing metadata (author, post ID, post URL, timestamp, reply count, repost count, like count, view count if available), followed by the thread body rendered in chronological order, with media referenced via relative `./media/<filename>` links.
- **G3.** A `media/` directory per thread holding the downloaded assets, named by post ID + index (e.g., `media/<post_id>_img1.jpg`, `media/<post_id>_vid1.mp4`).
- **G4.** A `README.md` in `scripts/twitter_threads/` documenting: prerequisites (which acquisition strategy, which optional deps), usage (CLI invocation), output layout, and the "copy to another repo" instructions (which files to copy, which deps to install).
- **G5.** The scripts must be runnable both as `python scripts/twitter_threads/<module>.py` (standalone) and `python -m scripts.twitter_threads.<module>` (from this repo). No imports from `src/` or `conductor/`.
## Functional Requirements
### FR1: Acquisition Strategy (the core design decision)
The scripts must support at least one acquisition strategy that works without paid X.com developer credentials. The track must implement and document the tradeoffs of:
- **Strategy A — `gallery-dl` (recommended default):** `gallery-dl` is a mature, actively-maintained tool that can download Twitter/X posts + threads + media. It handles auth via cookies exported from the browser (the `cookies.txt` pattern) or via guest access where available. It produces JSON metadata + downloaded media files. The scripts wrap `gallery-dl` as a subprocess (same pattern as `scripts/video_analysis/download_video.py` wraps `yt-dlp`).
- **Strategy B — `snscrape` (legacy fallback):** `snscrape` was the standard pre-2023 tool but is increasingly broken against X.com. Document it as a fallback for older content (pre-rename Twitter) but do not depend on it as the primary path.
- **Strategy C — manual export + post-processing (the no-network path):** the user saves the thread HTML via the browser (Ctrl+S, or a single-page archive), and the scripts parse the local HTML file to extract post text + media URLs, then download the media via `urllib` (no X.com auth needed for the media CDN URLs, which are public). This is the most resilient path if `gallery-dl` breaks.
- **Strategy D — `tweepy` / `twitter-api-python` (the paid-API path):** documented as the "if you have developer credentials" option, but NOT implemented as the default. The track notes it in the README as an alternative acquisition backend.
**Decision:** implement Strategy A (`gallery-dl` subprocess wrapper) as the primary path, with Strategy C (local HTML parse) as a fallback for when `gallery-dl` fails. Document Strategy B and Strategy D in the README as alternatives.
### FR2: `fetch_thread.py` — Thread Acquisition
- Input: a single tweet/X post URL (e.g., `https://x.com/<user>/status/<id>`) OR a local HTML file path.
- **URL normalization (added):** strip any query-string suffix before acquisition. X.com quote-share URLs carry a `?s=20` (or similar) param that is quote-share context, not part of the post identity. `fetch_thread.py` must parse the URL with `urllib.parse.urlparse` + reconstruct without the query string before passing to `gallery-dl`. This is a one-line `urlunparse((scheme, netloc, path, params, "", fragment))` call.
- If URL: invoke `gallery-dl` with the URL + `--write-metadata` + `--dump-json` flags to acquire the post + its thread context + media. Capture the JSON output. If `gallery-dl` fails (auth wall, rate limit), emit a `Result.err` with an `ErrorInfo` and document the fallback (Strategy C).
- If local HTML: parse the HTML to extract the post text, the post ID, the author handle, the timestamp, and the media URLs. Use `html.parser` from the stdlib (no BeautifulSoup dependency — standalone requirement).
- Output: a `ThreadData` dataclass (typed, frozen, slots) holding: the root post + the chain of replies (the thread), each post being a `PostData` dataclass (id, author, handle, text, timestamp, media_urls, reply_to_id, quote_of_id, metrics).
- Returns `Result[ThreadData, ErrorInfo]`.
### FR3: `download_media.py` — Media Download
- Input: a `ThreadData` instance + an output directory path.
- For each `PostData` in the thread, for each `media_url` in `post.media_urls`: download the file via `urllib.request.urlopen` (stdlib; no `requests` dependency). Name the file `<post_id>_<index><ext>` where ext is derived from the URL or Content-Type header.
- Handle the 4 media types X.com serves: images (`.jpg`, `.png`), videos (`.mp4`), GIFs (served as `.mp4` after the GIF-to-mp4 conversion X.com does), and inline images in quote-tweets.
- Returns `Result[list[Path], ErrorInfo]` — the list of downloaded file paths.
- Idempotent: if a file already exists at the target path with the same byte size, skip the download.
### FR4: `render_markdown.py` — Markdown Emission
- Input: a `ThreadData` instance + the list of downloaded media paths + an output `.md` path.
- Output schema:
```
---
title: "<thread title or first post's first line>"
author: "<display name>"
handle: "@<handle>"
post_url: "<root post URL>"
post_id: "<root post ID>"
timestamp: "<ISO 8601>"
post_count: <N>
reply_count: <N>
repost_count: <N>
like_count: <N>
view_count: <N or null>
---
# @<handle> — <thread title>
## Post 1 (<ISO timestamp>)
<post text, preserving line breaks>
[Media 1](./media/<post_id>_img1.jpg)
[Media 2](./media/<post_id>_vid1.mp4)
## Post 2 (<ISO timestamp>) — reply to Post 1
<post text>
...
```
- Quote-tweets: rendered as a nested blockquote with a link to the quoted post.
- Reply chains: each post in the thread is a `## Post N` section, in chronological order.
- Media: linked via relative `./media/<filename>` paths so the Markdown renders correctly when the `.md` + `media/` directory are moved together.
- Returns `Result[Path, ErrorInfo]` — the output `.md` path.
### FR5: `error_types.py` — Shared Error Type
- Copies the shape of `scripts/video_analysis/error_types.py`: `ErrorInfo` dataclass (frozen, slots) + `make_error` factory. Standalone — no import from `scripts.video_analysis`.
### FR6: `__init__.py` — Namespace Docstring
- Mirrors `scripts/video_analysis/__init__.py`: a docstring documenting the namespace, the per-module responsibilities, and the standalone-usage note.
### FR7: `README.md` — Standalone Usage Doc
- Prerequisites: `gallery-dl` (install via `pip install gallery-dl`), a `cookies.txt` exported from the browser (for auth-gated content), Python 3.11+.
- Usage:
```bash
# From this repo:
uv run python -m scripts.twitter_threads.fetch_thread "https://x.com/<user>/status/<id>" --output ./thread_output/
# Standalone (copied to another repo):
python scripts/twitter_threads/fetch_thread.py "https://x.com/<user>/status/<id>" --output ./thread_output/
```
- Output layout:
```
thread_output/
<post_id>/
thread.md
media/
<post_id>_img1.jpg
<post_id>_vid1.mp4
```
- "Copy to another repo" instructions: copy the entire `scripts/twitter_threads/` directory + the `error_types.py` dependency (which is self-contained). No `src/` imports. The only external dep is `gallery-dl` (installable via pip).
### FR8: Tests
- `tests/test_twitter_threads_render.py` — tests for `render_markdown.py` (the pure-function module; no network). Construct a `ThreadData` fixture, render to Markdown, assert the YAML front-matter + the per-post sections + the media links.
- `tests/test_twitter_threads_fetch.py` — tests for `fetch_thread.py`'s local-HTML-parse path (Strategy C). Feed a small fixture HTML file, assert the parsed `ThreadData`.
- `tests/test_twitter_threads_media.py` — tests for `download_media.py`'s naming + idempotency logic. Mock the HTTP fetch (no network in tests); assert file naming + skip-if-exists behavior.
- Per `conductor/code_styleguides/error_handling.md`, all tests assert on `Result.ok` / `Result.err` rather than raising.
## Non-Functional Requirements
- **Standalone:** zero imports from `src/`, `conductor/`, `scripts.video_analysis`, or any project-internal module. The `scripts/twitter_threads/` directory must be copy-pasteable to another repo with only `gallery-dl` as an external dep.
- **Dependency-light:** stdlib only for the script internals (`urllib`, `html.parser`, `json`, `pathlib`, `dataclasses`, `subprocess`). `gallery-dl` is invoked as a subprocess, not imported. No `requests`, no `BeautifulSoup`, no `tweepy` in the default path.
- **1-space indentation** per `conductor/code_styleguides/python.md` §1.
- **Type hints** on all public functions.
- **Result[T, ErrorInfo]** return types per `conductor/code_styleguides/error_handling.md`. No bare exceptions in the public API.
- **No comments** in implementation code per `conductor/code_styleguides/python.md` (docstrings allowed; inline comments banned).
- **CRLF line endings** on Windows (the repo default).
- **File size convention** per `AGENTS.md`: the scripts go in `scripts/twitter_threads/` (scripts are namespace-isolated by directory). No new `src/` files.
## Architecture Reference
- **`scripts/video_analysis/`** — the precedent pipeline (the pattern this track mirrors).
- **`scripts/video_analysis/download_video.py`** — the subprocess-wrapper pattern (`yt-dlp` → `gallery-dl` analog).
- **`scripts/video_analysis/error_types.py`** — the `ErrorInfo` shape (copied, not imported).
- **`conductor/code_styleguides/error_handling.md`** — the Result[T] + NIL_T sentinel pattern.
- **`conductor/code_styleguides/python.md`** §1 — 1-space indentation + type hints + no comments.
- **`conductor/code_styleguides/data_oriented_design.md`** §8.5 — typed `@dataclass(frozen=True, slots=True)` for `ThreadData` / `PostData`. No `dict[str, Any]` in the script internals.
## Out of Scope
- **Integration with the Manual Slop GUI.** The scripts are standalone; no `gui_2.py` or `app_controller.py` changes.
- **Integration with the RAG subsystem.** The output Markdown could be RAG-indexed, but that's a future track.
- **Integration with the MMA engine.** The scripts are not MMA tools; they're standalone analysis tooling.
- **Real-time streaming / polling.** The scripts are one-shot extractors, not stream listeners.
- **Twitter API v2 paid endpoints.** Documented in the README as an alternative; not implemented as the default.
- **`snscrape` as a hard dependency.** Documented as a legacy fallback; not installed by default.
- **Browser automation (Selenium/Playwright).** Not used. The local-HTML-parse fallback (Strategy C) uses the user's manually-saved HTML, not a headless browser.
- **Cross-post aggregation (Twitter + Mastodon + Bluesky).** Twitter/X only for this track.
## Track Structure
This is a single track (no children). The work is small enough to fit in one plan. If the user later wants a "Twitter thread analysis campaign" (the analog of the video analysis campaign — many threads, cross-thread synthesis), that's a separate future track that consumes this tooling.
## Verification
- **VC1:** `scripts/twitter_threads/` exists with 5 files: `__init__.py`, `error_types.py`, `fetch_thread.py`, `download_media.py`, `render_markdown.py`.
- **VC2:** `scripts/twitter_threads/README.md` exists and documents prerequisites, usage, output layout, and the "copy to another repo" instructions.
- **VC3:** `tests/test_twitter_threads_render.py` passes: given a `ThreadData` fixture, `render_markdown.py` produces the expected Markdown (YAML front-matter + per-post sections + media links).
- **VC4:** `tests/test_twitter_threads_fetch.py` passes: given a small fixture HTML file, `fetch_thread.py` (Strategy C path) produces the expected `ThreadData`.
- **VC5:** `tests/test_twitter_threads_media.py` passes: `download_media.py` produces correctly-named files and skips existing files.
- **VC6:** `python scripts/twitter_threads/fetch_thread.py --help` works (standalone invocation, no `src/` imports).
- **VC7:** The scripts contain zero imports from `src/`, `conductor/`, or `scripts.video_analysis`. Verified via `grep -r "from src\." scripts/twitter_threads/` returning nothing + `grep -r "import src\." scripts/twitter_threads/` returning nothing.
## See Also
- **`conductor/tracks/video_analysis_campaign_2_20260627/spec.md`** — the sibling track (video analysis; the pattern this track mirrors).
- **`scripts/video_analysis/`** — the precedent pipeline.
- **`conductor/code_styleguides/python.md`** §1 — 1-space indentation.
- **`conductor/code_styleguides/error_handling.md`** — Result[T] pattern.
- **`conductor/code_styleguides/data_oriented_design.md`** §8.5 — typed dataclasses, no `dict[str, Any]`.
conductor/tracks/twitter_threads_extraction_20260705/state.toml
+84
View File
@@ -0,0 +1,84 @@
# Track state for twitter_threads_extraction_20260705
# Initialized by Tier 1 Orchestrator on 2026-07-05.
# Standalone tooling track: scripts/twitter_threads/ + tests + README. No src/ changes.
[meta]
track_id = "twitter_threads_extraction_20260705"
name = "Twitter/X Thread Extraction Tooling"
status = "active"
current_phase = 0
last_updated = "2026-07-05"
[blocked_by]
# None. Standalone scripts + tests; no dependency on other tracks.
[blocks]
# None. The tooling may be consumed by a future "Twitter thread analysis campaign" track.
[phases]
phase_1 = { status = "pending", checkpointsha = "", name = "Scaffold + Error Types + Data Classes" }
phase_2 = { status = "pending", checkpointsha = "", name = "render_markdown.py (TDD-first; pure function)" }
phase_3 = { status = "pending", checkpointsha = "", name = "download_media.py (stdlib urllib + idempotent)" }
phase_4 = { status = "pending", checkpointsha = "", name = "fetch_thread.py (gallery-dl subprocess + html.parser fallback)" }
phase_5 = { status = "pending", checkpointsha = "", name = "README + End-to-End CLI + Verification" }
[tasks]
t1_1 = { status = "pending", commit_sha = "", description = "Create scripts/twitter_threads/ + __init__.py (namespace docstring)" }
t1_2 = { status = "pending", commit_sha = "", description = "Write error_types.py (ErrorInfo + make_error; standalone, no import from scripts.video_analysis)" }
t1_3 = { status = "pending", commit_sha = "", description = "Write typed dataclasses (PostData, PostMetrics, ThreadData; frozen, slots)" }
t1_4 = { status = "pending", commit_sha = "", description = "Write tests/test_twitter_threads_types.py (TDD red-first)" }
t1_5 = { status = "pending", commit_sha = "", description = "Commit Phase 1" }
t2_1 = { status = "pending", commit_sha = "", description = "Write failing tests for render_markdown.py (5+ tests: single post, thread, quote-tweet, media links, metrics, title)" }
t2_2 = { status = "pending", commit_sha = "", description = "Implement render_markdown.py (YAML front-matter + per-post sections + media links + quote-tweet blockquote)" }
t2_3 = { status = "pending", commit_sha = "", description = "Commit Phase 2" }
t3_1 = { status = "pending", commit_sha = "", description = "Write failing tests for download_media.py (4+ tests: naming, video naming, idempotent, http error)" }
t3_2 = { status = "pending", commit_sha = "", description = "Implement download_media.py (stdlib urllib; idempotent; typed naming <post_id>_<kind><index>.<ext>)" }
t3_3 = { status = "pending", commit_sha = "", description = "Commit Phase 3" }
t4_1 = { status = "pending", commit_sha = "", description = "Write failing tests for fetch_thread.py Strategy C (local HTML parse; 4+ tests)" }
t4_2 = { status = "pending", commit_sha = "", description = "Implement fetch_thread_from_html (html.parser.HTMLParser subclass; stdlib only)" }
t4_3 = { status = "pending", commit_sha = "", description = "Implement fetch_thread_from_url (gallery-dl subprocess wrapper; --dump-json + --write-metadata)" }
t4_4 = { status = "pending", commit_sha = "", description = "Implement CLI dispatch + --help (argparse; URL or local HTML path input)" }
t4_5 = { status = "pending", commit_sha = "", description = "Commit Phase 4" }
t5_1 = { status = "pending", commit_sha = "", description = "Write README.md (prerequisites, usage, output layout, copy-to-another-repo, Strategy B/D alternatives)" }
t5_2 = { status = "pending", commit_sha = "", description = "Verify standalone requirement (VC6 + VC7: --help works; zero src/conductor/video_analysis imports)" }
t5_3 = { status = "pending", commit_sha = "", description = "End-to-end smoke test (manual; user provides a real X.com URL + cookies.txt)" }
t5_4 = { status = "pending", commit_sha = "", description = "Run the full test suite for the new tests (batched verification)" }
t5_5 = { status = "pending", commit_sha = "", description = "Commit Phase 5 + README" }
t5_6 = { status = "pending", commit_sha = "", description = "Conductor — User Manual Verification (PAUSE for user confirmation)" }
t5_7 = { status = "pending", commit_sha = "", description = "Mark track complete + write TRACK_COMPLETION_twitter_threads_extraction_20260705.md" }
[verification]
vc1_package_exists = false
vc2_readme_exists = false
vc3_render_tests_pass = false
vc4_fetch_tests_pass = false
vc5_media_tests_pass = false
vc6_standalone_help_works = false
vc7_zero_internal_imports = false
vc8_corpus_extraction_complete = false
[reference_project]
path = "C:\\projects\\forth\\bootslop"
purpose = "The Twitter thread extraction tooling feeds reference material into bootslop's scripts and reference-generation pipeline."
corpus_urls = [
"https://x.com/NOTimothyLottes/status/1757198624818168210",
"https://x.com/NOTimothyLottes/status/1653570742762479620",
"https://x.com/NOTimothyLottes/status/1917646466417381426",
"https://x.com/NOTimothyLottes/status/1917645859791200562",
"https://x.com/NOTimothyLottes/status/1917644904055910502",
"https://x.com/NOTimothyLottes/status/1917642786804785230",
"https://x.com/VPCOMPRESSB/status/1991383117571957052",
"https://x.com/VPCOMPRESSB/status/1987744335333622188",
]
corpus_output_dir = "tests/artifacts/twitter_threads_corpus/"
acceptance_criterion = "All 8 threads extract cleanly to thread.md + media/ pairs; the corpus is handed off to bootslop. The ?s=20 query suffix on the @VPCOMPRESSB URLs must be stripped by fetch_thread.py before acquisition."
[standalone_requirement]
rationale = "Per user directive: scripts must be standalone tooling so they don't have lots of dependencies with the repo's codebase and could be utilized in others."
enforcement = "VC7 — zero imports from src/, conductor/, or scripts.video_analysis. Verified via grep."
copyable = "The scripts/twitter_threads/ directory is copy-pasteable to another repo with only gallery-dl as an external dep."
[acquisition_strategy]
primary = "Strategy A: gallery-dl subprocess wrapper (URL input; --dump-json + --write-metadata)"
fallback = "Strategy C: local HTML parse via html.parser.HTMLParser (user saves HTML via browser; script parses offline)"
documented_alternatives = "Strategy B: snscrape (legacy; documented in README); Strategy D: tweepy/twitter-api-python (paid API; documented in README)"