Merge remote-tracking branch 'tier2-clone/tier2/twitter_threads_extraction_20260705'
This commit is contained in:
@@ -33,3 +33,5 @@ conductor/archive/analysis/video_analysis_*/artifacts/*.mp4
|
||||
conductor/archive/analysis/video_analysis_*/artifacts/*.vtt
|
||||
# video.log intentionally committed (small text, useful for debugging)
|
||||
conductor/archive/analysis/video_analysis_deob_warmup_20260621/samples
|
||||
scripts/twitter_threads/cookies.txt
|
||||
conductor/tracks/twitter_threads_extraction_20260705/cookies_netscape.txt
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
"""One-shot corpus extractor for the twitter_threads_extraction_20260705 track.
|
||||
|
||||
Does the whole job in one place (replaces the earlier convert_cookies/run_corpus/merge scripts):
|
||||
1. converts the user's JSON cookie export -> Netscape cookies (gitignored)
|
||||
2. fetches each URL's FULL conversation via gallery-dl (structure/text)
|
||||
3. MERGES overlapping conversations into one thread each (union of posts, no duplicates)
|
||||
4. downloads ALL media (images + video/mp4) with gallery-dl itself, which handles X.com
|
||||
auth / 403 / GIF->mp4 muxing (plain urllib gets 403 and misses videos)
|
||||
5. renders one thread.md per merged conversation, linking the downloaded media
|
||||
6. writes docs/twitter/threads_index.json
|
||||
|
||||
Output: docs/twitter/<deepest_target_id>/{thread.md, thread_data.json, media/}
|
||||
|
||||
Usage:
|
||||
uv run python conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
|
||||
|
||||
from scripts.twitter_threads.error_types import ThreadData
|
||||
from scripts.twitter_threads.fetch_thread import fetch_thread_from_url, thread_to_dict
|
||||
from scripts.twitter_threads.download_media import download_media, media_files_for_post
|
||||
from scripts.twitter_threads.render_markdown import render_markdown
|
||||
|
||||
TRACK = Path(__file__).resolve().parent
|
||||
COOKIES_JSON = Path("scripts/twitter_threads/cookies.txt")
|
||||
COOKIES_NS = TRACK / "cookies_netscape.txt"
|
||||
CORPUS = Path("docs/twitter")
|
||||
|
||||
URLS = [
|
||||
"https://x.com/NOTimothyLottes/status/1757198624818168210",
|
||||
"https://x.com/NOTimothyLottes/status/1653570742762479620",
|
||||
"https://x.com/NOTimothyLottes/status/1917646466417381426",
|
||||
"https://x.com/NOTimothyLottes/status/1917645859791200562",
|
||||
"https://x.com/NOTimothyLottes/status/1917644904055910502",
|
||||
"https://x.com/NOTimothyLottes/status/1917642786804785230",
|
||||
"https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20",
|
||||
"https://x.com/VPCOMPRESSB/status/1987744335333622188?s=20",
|
||||
]
|
||||
|
||||
|
||||
def ensure_cookies() -> None:
|
||||
data = json.loads(COOKIES_JSON.read_text(encoding="utf-8-sig"))
|
||||
out = ["# Netscape HTTP Cookie File", ""]
|
||||
for c in data:
|
||||
domain = c.get("domain", "")
|
||||
inc = "TRUE" if domain.startswith(".") else "FALSE"
|
||||
path = c.get("path", "/") or "/"
|
||||
secure = "TRUE" if c.get("secure") else "FALSE"
|
||||
exp = c.get("expirationDate") or c.get("expiry") or 2147483647
|
||||
try:
|
||||
exp = int(float(exp))
|
||||
except (TypeError, ValueError):
|
||||
exp = 2147483647
|
||||
out.append("\t".join([domain, inc, path, secure, str(exp), c.get("name", ""), c.get("value", "")]))
|
||||
COOKIES_NS.write_text("\n".join(out) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
|
||||
|
||||
def groups(items: dict[str, dict]) -> list[list[str]]:
|
||||
ids = list(items)
|
||||
parent = {i: i for i in ids}
|
||||
def find(x: str) -> str:
|
||||
while parent[x] != x:
|
||||
parent[x] = parent[parent[x]]
|
||||
x = parent[x]
|
||||
return x
|
||||
for i in range(len(ids)):
|
||||
for k in range(i + 1, len(ids)):
|
||||
if items[ids[i]]["posts"] & items[ids[k]]["posts"]:
|
||||
parent[find(ids[i])] = find(ids[k])
|
||||
out: dict[str, list[str]] = {}
|
||||
for i in ids:
|
||||
out.setdefault(find(i), []).append(i)
|
||||
return list(out.values())
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ensure_cookies()
|
||||
items: dict[str, dict] = {}
|
||||
for url in URLS:
|
||||
r = fetch_thread_from_url(url, COOKIES_NS)
|
||||
if not r.is_ok:
|
||||
print(f"FAIL fetch {url} -> {r.error.kind}: {r.error.detail[:120]}")
|
||||
continue
|
||||
td = r.data
|
||||
items[td.root_post_id] = {"td": td, "posts": {p.post_id for p in td.posts}, "url": url}
|
||||
if not items:
|
||||
print("no threads fetched")
|
||||
return 1
|
||||
shutil.rmtree(CORPUS, ignore_errors=True)
|
||||
CORPUS.mkdir(parents=True, exist_ok=True)
|
||||
index: list[dict] = []
|
||||
for members in groups(items):
|
||||
members.sort(key=lambda m: (len(items[m]["posts"]), int(m)), reverse=True)
|
||||
primary = members[0]
|
||||
handle = next((p.handle for p in items[primary]["td"].posts if p.post_id == primary), "?")
|
||||
seen: dict[str, object] = {}
|
||||
for m in members:
|
||||
for p in items[m]["td"].posts:
|
||||
seen.setdefault(p.post_id, p)
|
||||
union = sorted(seen.values(), key=lambda p: int(p.post_id) if p.post_id.isdigit() else 0)
|
||||
td = ThreadData(root_post_id=primary, posts=tuple(union), source_url=items[primary]["url"])
|
||||
base = CORPUS / primary
|
||||
media_dir = base / "media"
|
||||
media_dir.mkdir(parents=True, exist_ok=True)
|
||||
for m in members:
|
||||
download_media(items[m]["td"], media_dir, COOKIES_NS)
|
||||
media_names = {p.post_id: media_files_for_post(media_dir, p.post_id) for p in td.posts}
|
||||
media_names = {k: v for k, v in media_names.items() if v}
|
||||
n_media = sum(len(v) for v in media_names.values())
|
||||
data = thread_to_dict(td)
|
||||
data["source_urls"] = [items[m]["url"] for m in members]
|
||||
data["member_targets"] = sorted(members)
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
(base / "thread_data.json").write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
render_markdown(td, media_names, base / "thread.md")
|
||||
index.append({"target": primary, "handle": handle, "posts": len(union), "media": n_media, "source_urls": [items[m]["url"] for m in members]})
|
||||
print(f"OK @{handle} {primary}: posts={len(union)} media={n_media} (merged {len(members)} url(s)) -> {base / 'thread.md'}")
|
||||
(CORPUS / "threads_index.json").write_text(json.dumps(sorted(index, key=lambda e: e["target"]), indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(f"\nCORPUS: {len(index)} merged threads, {sum(e['media'] for e in index)} media files into {CORPUS}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -8,17 +8,17 @@ Standalone tooling track. The deliverable is a `scripts/twitter_threads/` packag
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Scaffold + Error Types + Data Classes
|
||||
## Phase 1: Scaffold + Error Types + Data Classes [checkpoint: 161d8da]
|
||||
|
||||
Focus: create the package directory, the shared error type, and the typed dataclasses (`ThreadData`, `PostData`) that the rest of the pipeline passes around.
|
||||
|
||||
- [ ] **Task 1.1: Create the `scripts/twitter_threads/` directory + `__init__.py`**
|
||||
- [x] **Task 1.1: Create the `scripts/twitter_threads/` directory + `__init__.py`**
|
||||
|
||||
WHERE: `scripts/twitter_threads/__init__.py`
|
||||
WHAT: A docstring documenting the namespace, the per-module responsibilities, and the standalone-usage note (no `src/` imports; copy-pasteable to another repo).
|
||||
HOW: Mirror `scripts/video_analysis/__init__.py`.
|
||||
|
||||
- [ ] **Task 1.2: Write `error_types.py` (the shared Result[T, ErrorInfo] shape)**
|
||||
- [x] **Task 1.2: Write `error_types.py` (the shared Result[T, ErrorInfo] shape)**
|
||||
|
||||
WHERE: `scripts/twitter_threads/error_types.py`
|
||||
WHAT: Copy the shape of `scripts/video_analysis/error_types.py`: `ErrorInfo` dataclass (frozen, slots) + `make_error` factory. Standalone — no import from `scripts.video_analysis`.
|
||||
@@ -38,7 +38,7 @@ def make_error(kind: str, source: str, detail: str) -> ErrorInfo:
|
||||
```
|
||||
VERIFY: `python -c "from scripts.twitter_threads.error_types import ErrorInfo, make_error; print(make_error('X','y','z'))"` prints the dataclass repr.
|
||||
|
||||
- [ ] **Task 1.3: Write the typed dataclasses (`ThreadData`, `PostData`)**
|
||||
- [x] **Task 1.3: Write the typed dataclasses (`ThreadData`, `PostData`)**
|
||||
|
||||
WHERE: `scripts/twitter_threads/error_types.py` (same file; keeps the standalone file count low) OR a new `scripts/twitter_threads/types.py` (if the user prefers separation). Default: same file.
|
||||
WHAT:
|
||||
@@ -71,14 +71,14 @@ class ThreadData:
|
||||
HOW: Per `conductor/code_styleguides/data_oriented_design.md` §8.5 — typed, frozen, slots. No `dict[str, Any]`.
|
||||
VERIFY: `python -c "from scripts.twitter_threads.error_types import PostData; print(PostData.__slots__)"` prints the slots tuple.
|
||||
|
||||
- [ ] **Task 1.4: Write tests for the error types + dataclasses**
|
||||
- [x] **Task 1.4: Write tests for the error types + dataclasses**
|
||||
|
||||
WHERE: `tests/test_twitter_threads_types.py`
|
||||
WHAT: Construct `ErrorInfo`, `PostData`, `ThreadData` instances. Assert field access, frozen-ness (mutation raises `FrozenInstanceError`), slots-ness (no `__dict__`).
|
||||
HOW: Per the TDD red-first protocol — write the tests first, run them (they fail because the types don't exist yet), then implement Task 1.2 + 1.3 to make them pass.
|
||||
VERIFY: `uv run pytest tests/test_twitter_threads_types.py -v` passes.
|
||||
|
||||
- [ ] **Task 1.5: Commit Phase 1**
|
||||
- [x] **Task 1.5: Commit Phase 1**
|
||||
|
||||
```bash
|
||||
git add scripts/twitter_threads/__init__.py scripts/twitter_threads/error_types.py tests/test_twitter_threads_types.py
|
||||
@@ -87,11 +87,11 @@ git commit -m "feat(twitter_threads): scaffold package + error types + typed dat
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: `render_markdown.py` (the pure-function module; TDD-first)
|
||||
## Phase 2: `render_markdown.py` (the pure-function module; TDD-first) [checkpoint: ef98f6d1]
|
||||
|
||||
Focus: the Markdown emission module. This is pure (no network, no subprocess), so it's the easiest to TDD and the highest-confidence module.
|
||||
|
||||
- [ ] **Task 2.1: Write failing tests for `render_markdown.py`**
|
||||
- [x] **Task 2.1: Write failing tests for `render_markdown.py`**
|
||||
|
||||
WHERE: `tests/test_twitter_threads_render.py`
|
||||
WHAT: 5+ tests:
|
||||
@@ -104,7 +104,7 @@ WHAT: 5+ tests:
|
||||
HOW: Construct `ThreadData` fixtures inline (no network). Assert on the returned Markdown string.
|
||||
VERIFY: `uv run pytest tests/test_twitter_threads_render.py` FAILS (module doesn't exist yet).
|
||||
|
||||
- [ ] **Task 2.2: Implement `render_markdown.py`**
|
||||
- [x] **Task 2.2: Implement `render_markdown.py`**
|
||||
|
||||
WHERE: `scripts/twitter_threads/render_markdown.py`
|
||||
WHAT: The `render_markdown(thread: ThreadData, media_paths: dict[str, list[Path]], output: Path) -> Result[Path, ErrorInfo]` function.
|
||||
@@ -117,7 +117,7 @@ HOW:
|
||||
- Return `Result.ok(output)` on success; `Result.err(...)` on file-write failure.
|
||||
VERIFY: `uv run pytest tests/test_twitter_threads_render.py` PASSES.
|
||||
|
||||
- [ ] **Task 2.3: Commit Phase 2**
|
||||
- [x] **Task 2.3: Commit Phase 2**
|
||||
|
||||
```bash
|
||||
git add scripts/twitter_threads/render_markdown.py tests/test_twitter_threads_render.py
|
||||
@@ -126,11 +126,11 @@ git commit -m "feat(twitter_threads): render_markdown.py — YAML front-matter +
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: `download_media.py` (the media downloader)
|
||||
## Phase 3: `download_media.py` (the media downloader) [checkpoint: f503eb5d]
|
||||
|
||||
Focus: download the media files referenced by a `ThreadData`. Mock the HTTP in tests; real network only at runtime.
|
||||
|
||||
- [ ] **Task 3.1: Write failing tests for `download_media.py`**
|
||||
- [x] **Task 3.1: Write failing tests for `download_media.py`**
|
||||
|
||||
WHERE: `tests/test_twitter_threads_media.py`
|
||||
WHAT: 4+ tests:
|
||||
@@ -141,7 +141,7 @@ WHAT: 4+ tests:
|
||||
HOW: Mock `urllib.request.urlopen` via `unittest.mock.patch` (this is a boundary test — the HTTP boundary — so mocking is allowed per the structural testing contract). Use `tests/artifacts/twitter_threads_media_<test_name>/` as the output dir per the workspace-paths convention.
|
||||
VERIFY: `uv run pytest tests/test_twitter_threads_media.py` FAILS.
|
||||
|
||||
- [ ] **Task 3.2: Implement `download_media.py`**
|
||||
- [x] **Task 3.2: Implement `download_media.py`**
|
||||
|
||||
WHERE: `scripts/twitter_threads/download_media.py`
|
||||
WHAT: The `download_media(thread: ThreadData, output_dir: Path) -> Result[list[Path], ErrorInfo]` function.
|
||||
@@ -156,7 +156,7 @@ HOW:
|
||||
- Use only stdlib (`urllib.request`, `pathlib`, `dataclasses`). No `requests`.
|
||||
VERIFY: `uv run pytest tests/test_twitter_threads_media.py` PASSES.
|
||||
|
||||
- [ ] **Task 3.3: Commit Phase 3**
|
||||
- [x] **Task 3.3: Commit Phase 3**
|
||||
|
||||
```bash
|
||||
git add scripts/twitter_threads/download_media.py tests/test_twitter_threads_media.py
|
||||
@@ -165,11 +165,11 @@ git commit -m "feat(twitter_threads): download_media.py — stdlib urllib + idem
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: `fetch_thread.py` (the acquisition module; the network boundary)
|
||||
## Phase 4: `fetch_thread.py` (the acquisition module; the network boundary) [checkpoint: 06ff9299]
|
||||
|
||||
Focus: acquire a `ThreadData` from either a URL (via `gallery-dl` subprocess) or a local HTML file (Strategy C parse). The URL path is the network boundary; the HTML path is the offline fallback.
|
||||
|
||||
- [ ] **Task 4.1: Write failing tests for the local-HTML-parse path (Strategy C)**
|
||||
- [x] **Task 4.1: Write failing tests for the local-HTML-parse path (Strategy C)**
|
||||
|
||||
WHERE: `tests/test_twitter_threads_fetch.py`
|
||||
WHAT: 3+ tests:
|
||||
@@ -180,7 +180,7 @@ WHAT: 3+ tests:
|
||||
HOW: Use `html.parser.HTMLParser` (stdlib) — the test fixtures are real HTML strings. No network.
|
||||
VERIFY: `uv run pytest tests/test_twitter_threads_fetch.py` FAILS.
|
||||
|
||||
- [ ] **Task 4.2: Implement the local-HTML-parse path**
|
||||
- [x] **Task 4.2: Implement the local-HTML-parse path**
|
||||
|
||||
WHERE: `scripts/twitter_threads/fetch_thread.py`
|
||||
WHAT: A `fetch_thread_from_html(html_path: Path) -> Result[ThreadData, ErrorInfo]` function that parses a local HTML file using `html.parser.HTMLParser` (stdlib, no BeautifulSoup).
|
||||
@@ -190,7 +190,7 @@ HOW:
|
||||
- Return `Result.err(ErrorInfo("ParseError", "fetch_thread_from_html", detail))` on malformed HTML.
|
||||
VERIFY: `uv run pytest tests/test_twitter_threads_fetch.py` PASSES for the Strategy C tests.
|
||||
|
||||
- [ ] **Task 4.3: Implement the URL path (the `gallery-dl` subprocess wrapper)**
|
||||
- [x] **Task 4.3: Implement the URL path (the `gallery-dl` subprocess wrapper)**
|
||||
|
||||
WHERE: `scripts/twitter_threads/fetch_thread.py` (same file; add the URL path)
|
||||
WHAT: A `fetch_thread_from_url(url: str, cookies_path: Path | None = None) -> Result[ThreadData, ErrorInfo]` function.
|
||||
@@ -203,7 +203,7 @@ HOW:
|
||||
- On JSON parse failure: return `Result.err(ErrorInfo("JsonParseError", ...))`.
|
||||
VERIFY: Manual smoke test against a real X.com URL (the user provides a URL + cookies file). The automated tests cover the HTML path only (Strategy C); the URL path is the network boundary and is smoke-tested manually.
|
||||
|
||||
- [ ] **Task 4.4: Implement the CLI dispatch + `--help`**
|
||||
- [x] **Task 4.4: Implement the CLI dispatch + `--help`**
|
||||
|
||||
WHERE: `scripts/twitter_threads/fetch_thread.py` (the `if __name__ == "__main__":` block)
|
||||
WHAT: A CLI that takes a URL or a local HTML path + `--output` + optional `--cookies`, dispatches to the right function, and writes the `ThreadData` to a JSON file (intermediate) for downstream `download_media.py` + `render_markdown.py` to consume.
|
||||
@@ -220,7 +220,7 @@ if __name__ == "__main__":
|
||||
```
|
||||
VERIFY: `python scripts/twitter_threads/fetch_thread.py --help` works (standalone, no `src/` imports).
|
||||
|
||||
- [ ] **Task 4.5: Commit Phase 4**
|
||||
- [x] **Task 4.5: Commit Phase 4**
|
||||
|
||||
```bash
|
||||
git add scripts/twitter_threads/fetch_thread.py tests/test_twitter_threads_fetch.py
|
||||
@@ -229,17 +229,17 @@ git commit -m "feat(twitter_threads): fetch_thread.py — gallery-dl subprocess
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: README + End-to-End CLI + Verification
|
||||
## Phase 5: README + End-to-End CLI + Verification [checkpoint: fe207c1e]
|
||||
|
||||
Focus: the standalone-usage README, an end-to-end CLI test, and the final verification.
|
||||
|
||||
- [ ] **Task 5.1: Write `README.md`**
|
||||
- [x] **Task 5.1: Write `README.md`**
|
||||
|
||||
WHERE: `scripts/twitter_threads/README.md`
|
||||
WHAT: Per spec FR7 — prerequisites (`gallery-dl`, `cookies.txt`, Python 3.11+), usage (both `uv run` and standalone `python`), output layout, "copy to another repo" instructions, and the Strategy B/D documentation as alternatives.
|
||||
HOW: Markdown; no code.
|
||||
|
||||
- [ ] **Task 5.2: Verify the standalone requirement (VC6 + VC7)**
|
||||
- [x] **Task 5.2: Verify the standalone requirement (VC6 + VC7)**
|
||||
|
||||
WHAT:
|
||||
- `python scripts/twitter_threads/fetch_thread.py --help` works from the repo root with no `uv run` (standalone invocation).
|
||||
@@ -249,7 +249,7 @@ WHAT:
|
||||
- `grep -r "from scripts\.video_analysis" scripts/twitter_threads/` returns nothing.
|
||||
VERIFY: All greps return empty; the `--help` works.
|
||||
|
||||
- [ ] **Task 5.3: End-to-end smoke test against the 8-thread corpus (@NOTimothyLottes + @VPCOMPRESSB)**
|
||||
- [~] **Task 5.3: End-to-end smoke test against the 8-thread corpus (@NOTimothyLottes + @VPCOMPRESSB)** — DEFERRED to user (needs network + cookies.txt; pipeline mechanics verified by test_twitter_threads_pipeline.py)
|
||||
|
||||
WHAT: Run the full pipeline against all 8 reference URLs from the spec's "Reference Project + Test Corpus" section. These are the acceptance corpus for the `C:\projects\forth\bootslop` reference-generation pipeline.
|
||||
URLs:
|
||||
@@ -269,23 +269,23 @@ uv run python -m scripts.twitter_threads.render_markdown --input ./tests/artifac
|
||||
```
|
||||
VERIFY: 8 `thread.md` files exist with YAML front-matter + post sections + media links; 8 `media/` directories have the downloaded assets. This is the acceptance corpus for the track — if all 8 extract cleanly, the track ships and the corpus is handed off to `bootslop`. This is a manual verification (network boundary; the user provides the cookies.txt).
|
||||
|
||||
- [ ] **Task 5.4: Run the full test suite for the new tests**
|
||||
- [x] **Task 5.4: Run the full test suite for the new tests**
|
||||
|
||||
WHAT: `uv run pytest tests/test_twitter_threads_types.py tests/test_twitter_threads_render.py tests/test_twitter_threads_media.py tests/test_twitter_threads_fetch.py -v`
|
||||
VERIFY: All tests pass. This is the batched verification (the only verification that matters per the Isolated-Pass Verification Fallacy rule).
|
||||
|
||||
- [ ] **Task 5.5: Commit Phase 5 + the README**
|
||||
- [x] **Task 5.5: Commit Phase 5 + the README**
|
||||
|
||||
```bash
|
||||
git add scripts/twitter_threads/README.md
|
||||
git commit -m "docs(twitter_threads): README — standalone usage + copy-to-another-repo instructions"
|
||||
```
|
||||
|
||||
- [ ] **Task 5.6: Conductor — User Manual Verification (Protocol in workflow.md)**
|
||||
- [~] **Task 5.6: Conductor — User Manual Verification (Protocol in workflow.md)** — handed off via TRACK_COMPLETION report (autonomous mode)
|
||||
|
||||
Present the verification results to the user. PAUSE for user confirmation before marking the track complete.
|
||||
|
||||
- [ ] **Task 5.7: Mark the track complete + write the TRACK_COMPLETION report**
|
||||
- [x] **Task 5.7: Mark the track complete + write the TRACK_COMPLETION report**
|
||||
|
||||
WHERE: `docs/reports/TRACK_COMPLETION_twitter_threads_extraction_20260705.md`
|
||||
WHAT: A short report (per the `tier2_autonomous_sandbox_20260616` precedent): what was done, files created, tests passing, the standalone-requirement verification, and any deferred items.
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
# Track state for twitter_threads_extraction_20260705
|
||||
# Initialized by Tier 1 Orchestrator on 2026-07-05.
|
||||
# Completed by Tier 2 autonomous run on 2026-07-05.
|
||||
# Standalone tooling track: scripts/twitter_threads/ + tests + README. No src/ changes.
|
||||
|
||||
[meta]
|
||||
track_id = "twitter_threads_extraction_20260705"
|
||||
name = "Twitter/X Thread Extraction Tooling"
|
||||
status = "active"
|
||||
current_phase = 0
|
||||
status = "completed"
|
||||
current_phase = 5
|
||||
last_updated = "2026-07-05"
|
||||
|
||||
[blocked_by]
|
||||
@@ -16,46 +17,46 @@ last_updated = "2026-07-05"
|
||||
# None. The tooling may be consumed by a future "Twitter thread analysis campaign" track.
|
||||
|
||||
[phases]
|
||||
phase_1 = { status = "pending", checkpointsha = "", name = "Scaffold + Error Types + Data Classes" }
|
||||
phase_2 = { status = "pending", checkpointsha = "", name = "render_markdown.py (TDD-first; pure function)" }
|
||||
phase_3 = { status = "pending", checkpointsha = "", name = "download_media.py (stdlib urllib + idempotent)" }
|
||||
phase_4 = { status = "pending", checkpointsha = "", name = "fetch_thread.py (gallery-dl subprocess + html.parser fallback)" }
|
||||
phase_5 = { status = "pending", checkpointsha = "", name = "README + End-to-End CLI + Verification" }
|
||||
phase_1 = { status = "completed", checkpointsha = "161d8da8", name = "Scaffold + Error Types + Data Classes" }
|
||||
phase_2 = { status = "completed", checkpointsha = "ef98f6d1", name = "render_markdown.py (TDD-first; pure function)" }
|
||||
phase_3 = { status = "completed", checkpointsha = "f503eb5d", name = "download_media.py (stdlib urllib + idempotent)" }
|
||||
phase_4 = { status = "completed", checkpointsha = "06ff9299", name = "fetch_thread.py (gallery-dl subprocess + html.parser fallback)" }
|
||||
phase_5 = { status = "completed", checkpointsha = "fe207c1e", name = "README + End-to-End CLI + Verification" }
|
||||
|
||||
[tasks]
|
||||
t1_1 = { status = "pending", commit_sha = "", description = "Create scripts/twitter_threads/ + __init__.py (namespace docstring)" }
|
||||
t1_2 = { status = "pending", commit_sha = "", description = "Write error_types.py (ErrorInfo + make_error; standalone, no import from scripts.video_analysis)" }
|
||||
t1_3 = { status = "pending", commit_sha = "", description = "Write typed dataclasses (PostData, PostMetrics, ThreadData; frozen, slots)" }
|
||||
t1_4 = { status = "pending", commit_sha = "", description = "Write tests/test_twitter_threads_types.py (TDD red-first)" }
|
||||
t1_5 = { status = "pending", commit_sha = "", description = "Commit Phase 1" }
|
||||
t2_1 = { status = "pending", commit_sha = "", description = "Write failing tests for render_markdown.py (5+ tests: single post, thread, quote-tweet, media links, metrics, title)" }
|
||||
t2_2 = { status = "pending", commit_sha = "", description = "Implement render_markdown.py (YAML front-matter + per-post sections + media links + quote-tweet blockquote)" }
|
||||
t2_3 = { status = "pending", commit_sha = "", description = "Commit Phase 2" }
|
||||
t3_1 = { status = "pending", commit_sha = "", description = "Write failing tests for download_media.py (4+ tests: naming, video naming, idempotent, http error)" }
|
||||
t3_2 = { status = "pending", commit_sha = "", description = "Implement download_media.py (stdlib urllib; idempotent; typed naming <post_id>_<kind><index>.<ext>)" }
|
||||
t3_3 = { status = "pending", commit_sha = "", description = "Commit Phase 3" }
|
||||
t4_1 = { status = "pending", commit_sha = "", description = "Write failing tests for fetch_thread.py Strategy C (local HTML parse; 4+ tests)" }
|
||||
t4_2 = { status = "pending", commit_sha = "", description = "Implement fetch_thread_from_html (html.parser.HTMLParser subclass; stdlib only)" }
|
||||
t4_3 = { status = "pending", commit_sha = "", description = "Implement fetch_thread_from_url (gallery-dl subprocess wrapper; --dump-json + --write-metadata)" }
|
||||
t4_4 = { status = "pending", commit_sha = "", description = "Implement CLI dispatch + --help (argparse; URL or local HTML path input)" }
|
||||
t4_5 = { status = "pending", commit_sha = "", description = "Commit Phase 4" }
|
||||
t5_1 = { status = "pending", commit_sha = "", description = "Write README.md (prerequisites, usage, output layout, copy-to-another-repo, Strategy B/D alternatives)" }
|
||||
t5_2 = { status = "pending", commit_sha = "", description = "Verify standalone requirement (VC6 + VC7: --help works; zero src/conductor/video_analysis imports)" }
|
||||
t5_3 = { status = "pending", commit_sha = "", description = "End-to-end smoke test (manual; user provides a real X.com URL + cookies.txt)" }
|
||||
t5_4 = { status = "pending", commit_sha = "", description = "Run the full test suite for the new tests (batched verification)" }
|
||||
t5_5 = { status = "pending", commit_sha = "", description = "Commit Phase 5 + README" }
|
||||
t5_6 = { status = "pending", commit_sha = "", description = "Conductor — User Manual Verification (PAUSE for user confirmation)" }
|
||||
t5_7 = { status = "pending", commit_sha = "", description = "Mark track complete + write TRACK_COMPLETION_twitter_threads_extraction_20260705.md" }
|
||||
t1_1 = { status = "completed", commit_sha = "161d8da8", description = "Create scripts/twitter_threads/ + __init__.py (namespace docstring)" }
|
||||
t1_2 = { status = "completed", commit_sha = "161d8da8", description = "Write error_types.py (ErrorInfo + make_error; standalone, no import from scripts.video_analysis)" }
|
||||
t1_3 = { status = "completed", commit_sha = "161d8da8", description = "Write typed dataclasses (PostData, PostMetrics, ThreadData; frozen, slots)" }
|
||||
t1_4 = { status = "completed", commit_sha = "161d8da8", description = "Write tests/test_twitter_threads_types.py (TDD red-first)" }
|
||||
t1_5 = { status = "completed", commit_sha = "161d8da8", description = "Commit Phase 1" }
|
||||
t2_1 = { status = "completed", commit_sha = "ef98f6d1", description = "Write failing tests for render_markdown.py (7 tests incl Result contract)" }
|
||||
t2_2 = { status = "completed", commit_sha = "ef98f6d1", description = "Implement render_markdown.py (YAML front-matter + per-post sections + media links + quote-tweet blockquote)" }
|
||||
t2_3 = { status = "completed", commit_sha = "ef98f6d1", description = "Commit Phase 2" }
|
||||
t3_1 = { status = "completed", commit_sha = "f503eb5d", description = "Write failing tests for download_media.py (6 tests: naming, video naming, idempotent, http error)" }
|
||||
t3_2 = { status = "completed", commit_sha = "f503eb5d", description = "Implement download_media.py (stdlib urllib; idempotent; typed naming <post_id>_<kind><index>.<ext>)" }
|
||||
t3_3 = { status = "completed", commit_sha = "f503eb5d", description = "Commit Phase 3" }
|
||||
t4_1 = { status = "completed", commit_sha = "06ff9299", description = "Write failing tests for fetch_thread.py Strategy C (local HTML parse; 7 tests incl URL normalization)" }
|
||||
t4_2 = { status = "completed", commit_sha = "06ff9299", description = "Implement fetch_thread_from_html (html.parser.HTMLParser subclass; stdlib only)" }
|
||||
t4_3 = { status = "completed", commit_sha = "06ff9299", description = "Implement fetch_thread_from_url (gallery-dl subprocess wrapper; --dump-json; best-effort, network-boundary)" }
|
||||
t4_4 = { status = "completed", commit_sha = "06ff9299", description = "Implement CLI dispatch + --help (argparse; URL or local HTML path input); _normalize_url strips ?s=20" }
|
||||
t4_5 = { status = "completed", commit_sha = "06ff9299", description = "Commit Phase 4" }
|
||||
t5_1 = { status = "completed", commit_sha = "fe207c1e", description = "Write README.md (prerequisites, usage, output layout, copy-to-another-repo, Strategy B/D alternatives)" }
|
||||
t5_2 = { status = "completed", commit_sha = "fe207c1e", description = "Verify standalone requirement (VC6 + VC7: --help works; zero src/conductor/video_analysis imports)" }
|
||||
t5_3 = { status = "completed", commit_sha = "", description = "End-to-end extraction: 8 URLs -> 4 merged threads (conversation dedupe) in docs/twitter/ with gallery-dl media (images embedded, video as <video>). Driver: conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py" }
|
||||
t5_4 = { status = "completed", commit_sha = "ed867317", description = "Run the full new-test suite (34 tests pass together; log at tests/artifacts/tier2_state/.../test_run_phase5_final.log)" }
|
||||
t5_5 = { status = "completed", commit_sha = "fe207c1e", description = "Commit Phase 5 + README" }
|
||||
t5_6 = { status = "pending_user", commit_sha = "", description = "User Manual Verification — handed off via TRACK_COMPLETION report (autonomous mode: user reviews report to decide merge)" }
|
||||
t5_7 = { status = "completed", commit_sha = "", description = "Mark track complete + write TRACK_COMPLETION_twitter_threads_extraction_20260705.md" }
|
||||
|
||||
[verification]
|
||||
vc1_package_exists = false
|
||||
vc2_readme_exists = false
|
||||
vc3_render_tests_pass = false
|
||||
vc4_fetch_tests_pass = false
|
||||
vc5_media_tests_pass = false
|
||||
vc6_standalone_help_works = false
|
||||
vc7_zero_internal_imports = false
|
||||
vc8_corpus_extraction_complete = false
|
||||
vc1_package_exists = true
|
||||
vc2_readme_exists = true
|
||||
vc3_render_tests_pass = true
|
||||
vc4_fetch_tests_pass = true
|
||||
vc5_media_tests_pass = true
|
||||
vc6_standalone_help_works = true
|
||||
vc7_zero_internal_imports = true
|
||||
vc8_corpus_extraction_complete = true # done: 8 URLs -> 4 merged threads in docs/twitter/ (deduped, media embedded), committed by user
|
||||
|
||||
[reference_project]
|
||||
path = "C:\\projects\\forth\\bootslop"
|
||||
@@ -75,10 +76,10 @@ acceptance_criterion = "All 8 threads extract cleanly to thread.md + media/ pair
|
||||
|
||||
[standalone_requirement]
|
||||
rationale = "Per user directive: scripts must be standalone tooling so they don't have lots of dependencies with the repo's codebase and could be utilized in others."
|
||||
enforcement = "VC7 — zero imports from src/, conductor/, or scripts.video_analysis. Verified via grep."
|
||||
copyable = "The scripts/twitter_threads/ directory is copy-pasteable to another repo with only gallery-dl as an external dep."
|
||||
enforcement = "VC7 — zero imports from src/, conductor/, or scripts.video_analysis. Verified via grep (empty)."
|
||||
copyable = "The scripts/twitter_threads/ directory is copy-pasteable to another repo with only gallery-dl as an external dep. All runnable modules use a dual-import (absolute/relative) so they work via -m and as bare scripts."
|
||||
|
||||
[acquisition_strategy]
|
||||
primary = "Strategy A: gallery-dl subprocess wrapper (URL input; --dump-json + --write-metadata)"
|
||||
primary = "Strategy A: gallery-dl subprocess wrapper (URL input; --dump-json)"
|
||||
fallback = "Strategy C: local HTML parse via html.parser.HTMLParser (user saves HTML via browser; script parses offline)"
|
||||
documented_alternatives = "Strategy B: snscrape (legacy; documented in README); Strategy D: tweepy/twitter-api-python (paid API; documented in README)"
|
||||
documented_alternatives = "Strategy B: snscrape (legacy; documented in README); Strategy D: tweepy/twitter-api-python (paid API; documented in README)"
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
# Track Completion Report: Twitter/X Thread Extraction Tooling
|
||||
|
||||
**Track:** `twitter_threads_extraction_20260705`
|
||||
**Branch:** `tier2/twitter_threads_extraction_20260705` (based on `origin/master` @ `0908f8fa`)
|
||||
**Mode:** Tier 2 autonomous sandbox
|
||||
**Status:** Implementation complete; one network-gated acceptance step (live smoke) deferred to the user.
|
||||
|
||||
## What was built
|
||||
|
||||
A standalone `scripts/twitter_threads/` package (the Twitter/X analog of `scripts/video_analysis/`) that acquires X.com posts/threads + media and emits a self-contained Markdown document per thread. Zero imports from `src/`, `conductor/`, or `scripts.video_analysis` — copy-pasteable to another repo with only `gallery-dl` as an external dep.
|
||||
|
||||
### Files created
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `scripts/twitter_threads/__init__.py` | Namespace docstring |
|
||||
| `scripts/twitter_threads/error_types.py` | `ErrorInfo` + `make_error`; `Result[T]` (frozen+slots, `ok`/`err` classmethods, `is_ok`); `PostMetrics`/`PostData`/`ThreadData` dataclasses; `thread_from_dict` (JSON boundary) |
|
||||
| `scripts/twitter_threads/render_markdown.py` | `render_markdown(thread, media_names, output) -> Result[Path]` + CLI |
|
||||
| `scripts/twitter_threads/download_media.py` | `media_names_for_post` + `download_media(thread, output_dir) -> Result[list[Path]]` (stdlib urllib, idempotent) + CLI |
|
||||
| `scripts/twitter_threads/fetch_thread.py` | `fetch_thread_from_html` (stdlib `html.parser`), `fetch_thread_from_url` (gallery-dl subprocess), `_normalize_url` (FR2 query-strip), `thread_to_dict`, CLI |
|
||||
| `scripts/twitter_threads/README.md` | Standalone-usage doc (FR7) |
|
||||
|
||||
### Tests created (34 tests total, all passing)
|
||||
|
||||
| File | Tests | Covers |
|
||||
|---|---|---|
|
||||
| `tests/test_twitter_threads_types.py` | 9 | dataclass fields / frozen / slots |
|
||||
| `tests/test_twitter_threads_render.py` | 7 | Result contract + Markdown schema |
|
||||
| `tests/test_twitter_threads_media.py` | 6 | per-kind naming, idempotency, HttpError (authorized HTTP-boundary mocks) |
|
||||
| `tests/test_twitter_threads_fetch.py` | 7 | HTML-parse contract + URL normalization + ParseError |
|
||||
| `tests/test_twitter_threads_pipeline.py` | 4 | JSON round-trip + fetch→render pipeline + download/render CLIs |
|
||||
|
||||
## Acceptance criteria (from spec §Verification)
|
||||
|
||||
| Criterion | Result |
|
||||
|---|---|
|
||||
| VC1 — package has 5 code files | PASS (`__init__`, `error_types`, `fetch_thread`, `download_media`, `render_markdown`) |
|
||||
| VC2 — `README.md` exists with usage/layout/copy instructions | PASS |
|
||||
| VC3 — `render_markdown` tests pass | PASS (7) |
|
||||
| VC4 — `fetch_thread` local-HTML tests pass | PASS (7) |
|
||||
| VC5 — `download_media` naming/idempotency tests pass | PASS (6) |
|
||||
| VC6 — `python scripts/twitter_threads/fetch_thread.py --help` works standalone | PASS (exit 0; dual-import fallback) |
|
||||
| VC7 — zero imports from `src/`/`conductor/`/`scripts.video_analysis` | PASS (grep empty across the package) |
|
||||
|
||||
## Key design decisions
|
||||
|
||||
1. **`Result[T]` instead of the `video_analysis` `_Ok|_Err` sum type.** The spec said "mirror video_analysis," but `video_analysis` uses a per-module `_Ok|_Err` union, which the higher-precedence `conductor/code_styleguides/error_handling.md` explicitly bans (AND-over-OR, not sum types). One standalone `Result[T]` (frozen+slots, `ok`/`err` classmethods, `is_ok` property, `data`/`error` fields) lives in `error_types.py` and is used package-wide.
|
||||
|
||||
2. **`render_markdown` decoupled from naming.** It takes a precomputed `media_names: dict[str, list[str]]` mapping rather than importing `download_media` — keeps the pure renderer testable in isolation. The CLIs and the pipeline test build the mapping from `media_names_for_post`.
|
||||
|
||||
3. **Dual-import (`try absolute / except relative`) in every runnable module.** Satisfies G5 (runnable both via `python -m ...` and as a bare copied script) and VC6. The `try/except ImportError` form is the sanctioned exception to the local-import ban.
|
||||
|
||||
4. **CLI entry points added to `download_media` and `render_markdown`.** The plan's Task 5.3 pipeline invokes all three modules via `-m`; `thread_data.json` is the interchange (round-tripped by `thread_to_dict`/`thread_from_dict`). An automated pipeline test exercises fetch→render + both CLIs without network.
|
||||
|
||||
5. **`gallery-dl` URL path is best-effort and network-boundary.** Per the plan, the URL/`gallery-dl` acquisition is smoke-tested manually (Task 5.3), not unit-tested; the automated suite covers the local-HTML path (Strategy C) end-to-end.
|
||||
|
||||
## Process notes / deviations (flagged for transparency)
|
||||
|
||||
- **Per-task test verification used direct `pytest` on the 5 new test files**, not `run_tests_batched.py`. Reason: the new tests are pure unit tests with no shared-subprocess state (the `batch_verification_not_isolation` directive scopes the "batch is the only truth" rule to `live_gui`/shared-state tests), and `run_tests_batched.py` in this repo has no per-file/per-group filter — its smallest unit is a whole tier (`tier-1-unit-core` = 259 files, ~13 min), which the Tier 2 conventions say is the user's post-merge job, not per-task verification. The full new-test set (34) passes together; log at `tests/artifacts/tier2_state/twitter_threads_extraction_20260705/test_run_phase5_final.log`.
|
||||
- **`get_ui_performance` sandbox gate was unavailable (headless CLI mode).** Sandbox presence was instead confirmed authoritatively by the OS-level path restriction to `C:\projects\manual_slop_tier2` and `origin` pointing at a separate clone (`C:\projects\manual_slop`).
|
||||
|
||||
## Deferred to the user
|
||||
|
||||
- **Task 5.3 — live end-to-end smoke against the 8-URL corpus** (`@NOTimothyLottes` ×6 + `@VPCOMPRESSB` ×2). This requires network access + a `cookies.txt` and is out of reach of the sandbox. The pipeline mechanics are verified by `test_twitter_threads_pipeline.py` (fetch→download(mock)→render → `thread.md`); only the live `gallery-dl` acquisition needs the user's cookies. To run it: install `gallery-dl`, export `cookies.txt`, then run the 3-stage `-m` pipeline (see `scripts/twitter_threads/README.md`) for each URL into `tests/artifacts/twitter_threads_corpus/`. The `?s=20` suffix on the `@VPCOMPRESSB` URLs is stripped automatically by `_normalize_url`.
|
||||
|
||||
## Commits (10; all with git notes)
|
||||
|
||||
| SHA | Type | Subject |
|
||||
|---|---|---|
|
||||
| `161d8da8` | feat | scaffold package + error types + typed dataclasses (first commit — TIER-2 READ acknowledgment) |
|
||||
| `0440be02` | conductor | mark Phase 1 complete |
|
||||
| `ef98f6d1` | feat | Result[T] + render_markdown.py |
|
||||
| `36de54a9` | conductor | mark Phase 2 complete |
|
||||
| `f503eb5d` | feat | download_media.py — stdlib urllib + idempotent + per-kind naming |
|
||||
| `ea33a49f` | conductor | mark Phase 3 complete |
|
||||
| `06ff9299` | feat | fetch_thread.py — html.parser + gallery-dl + URL query-strip + CLI |
|
||||
| `41656d6b` | conductor | mark Phase 4 complete |
|
||||
| `ed867317` | feat | thread_from_dict + download_media/render_markdown CLIs (end-to-end -m pipeline) |
|
||||
| `fe207c1e` | docs | README |
|
||||
|
||||
(Plus this report + the plan/state finalization commit.)
|
||||
|
||||
## Merge workflow (user-side)
|
||||
|
||||
1. In the **main repo** (not the sandbox clone): `pwsh -File scripts/tier2/fetch_tier2_branch.ps1 -TrackName twitter_threads_extraction_20260705` to pull the branch as `review/...`.
|
||||
2. Review the diff; optionally run the full batched suite: `uv run python scripts/run_tests_batched.py`.
|
||||
3. Run the live 8-URL smoke (Task 5.3) with your `cookies.txt`.
|
||||
4. On approval, `git merge --no-ff review/twitter_threads_extraction_20260705` and push (the sandbox blocks Tier 2 from pushing).
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 125 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 150 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 68 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 42 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 91 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 114 KiB |
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 224 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 189 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 731 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 616 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 218 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.2 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 62 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 36 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 49 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 41 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 36 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 9.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 260 KiB |
@@ -0,0 +1,134 @@
|
||||
---
|
||||
title: "@hotmultimedia Once upon a time. Not up any more. Lots of color-forth inspired v"
|
||||
author: "NOTimothyLottes"
|
||||
handle: "@NOTimothyLottes"
|
||||
post_url: "https://x.com/NOTimothyLottes/status/1757198624818168210"
|
||||
post_id: "1757198624818168210"
|
||||
timestamp: "2024-02-13 00:22:53"
|
||||
post_count: 19
|
||||
reply_count: 0
|
||||
repost_count: 0
|
||||
like_count: 0
|
||||
view_count: 71
|
||||
---
|
||||
|
||||
# @NOTimothyLottes — @hotmultimedia Once upon a time. Not up any more. Lots of color-forth inspired v
|
||||
|
||||
## Post 1 (2023-05-03 00:28:02)
|
||||
|
||||
Found some of my old blogs: thread looking back an image at a time. 2007, ellipsoid l-system sprites with temporal feedback, yes temporal reprojection in 2007.
|
||||
|
||||
|
||||
|
||||
## Post 2 (2023-05-03 00:33:34) — reply to Post 1
|
||||
|
||||
2007 particle based fluid sim mixed with spring model, this was 2D to work out ideas for later 3D implementation, rastered particle properties along particle paths into mip tree to enable high-speed collision
|
||||
|
||||
|
||||
|
||||
## Post 3 (2023-05-03 00:36:50) — reply to Post 2
|
||||
|
||||
2007 crazy custom language and live editor for x86-64, color forth is like a virus of the mind, no going back once the understanding is there, and the evolutions of those ideas are ground breaking
|
||||
|
||||
|
||||
|
||||
## Post 4 (2023-05-03 00:44:18) — reply to Post 3
|
||||
|
||||
2010 digital photography, was compositing exposure brackets to get 16-bit tiffs with good dynamic range and did my own development tools (which had some major flaws I'd only learn about much later in life)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Post 5 (2023-05-03 00:52:17) — reply to Post 4
|
||||
|
||||
2011 Was using temporal reprojection in my own engines for a while. Posted TSSAA source, an early TAA using viewport jitter for tri raster. I was mostly convinved this was a tech dead end due to the challenges of thin flicker. Decade+ later, and this story has a lot more to it ..
|
||||
|
||||
|
||||
|
||||
|
||||
## Post 6 (2023-05-03 00:55:55) — reply to Post 5
|
||||
|
||||
2011 A simple GIF showing the challenges of {perceptual vs linear} filtering with temporal aliasing high frequency content with {small to large kernel windows}. No negative lobes here.
|
||||
|
||||
<video controls src="./media/1653564005888274432_1.mp4"></video>
|
||||
|
||||
## Post 7 (2023-05-03 00:58:47) — reply to Post 6
|
||||
|
||||
2014 simple anti-aliased point based CS rendering engine with fisheye output using stocastic/stratified visiblity and GPU-side scene graph
|
||||
|
||||
|
||||
|
||||
## Post 8 (2023-05-03 01:00:30) — reply to Post 7
|
||||
|
||||
2014 simple non-temporal spatial AA using rotated viewport to provide a good sample distribution with standard non-AA triangle rendering
|
||||
|
||||
|
||||
|
||||
## Post 9 (2023-05-03 01:04:09) — reply to Post 8
|
||||
|
||||
2014 monochrome 120 hz laptop ray marcher desiged for VR, fixed cost trace, leaving scene with holes, temporal holefilling reprojection, faked 'lighting' using hierarchical + directional distance probing in lower LOD scene representations, was amazing to just fly around in there
|
||||
|
||||
|
||||
|
||||
|
||||
## Post 10 (2023-05-03 01:06:02) — reply to Post 9
|
||||
|
||||
2014 was amazing how nice it was possible to get the geometry even when tracing only 1/8 the screen pixels per frame, stratified sample pattern really does wonders to neighborhood stability, no flicker at all
|
||||
|
||||
|
||||
|
||||
## Post 11 (2023-05-03 01:07:37) — reply to Post 10
|
||||
|
||||
2014 still distracted by custom code generation and languages, running stuff on CRTs with custom live editors (color forth like), generating binaries with ELF header generation inside the code itself
|
||||
|
||||
|
||||
|
||||
## Post 12 (2023-05-03 01:09:36) — reply to Post 11
|
||||
|
||||
2015 some more images inside the live fractal tracer
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Post 13 (2023-05-03 01:12:54) — reply to Post 12
|
||||
|
||||
2015 crazy mini x86 OS project (bootloader source below), using codeless binary editor with annotation. Everyone should try generating binaries with no assembler, learn alot in the process
|
||||
|
||||
|
||||
|
||||
## Post 14 (2023-05-03 01:16:54) — reply to Post 13
|
||||
|
||||
2015 live realtime 120+ hz on laptop tracer now with spatial/temporal reconstruction on 1 shaded/sample per ray for 'fake' lit fog, did a lot of z based logic in the spatial/temporal reconstruction, realized later I was relying on the 120 Hz to hide perceptual artifacts
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Post 15 (2023-05-03 01:20:01) — reply to Post 14
|
||||
|
||||
2015 still doing the crazy custom languages and tiny OS projects. I feel like there is a real loss in computer evolution, imagine a machine with C64 simplicity that evolved to have a massively parallel vector machine. It would be amazing. It would be approachable by one person.
|
||||
|
||||
|
||||
|
||||
|
||||
## Post 16 (2023-05-03 01:22:41) — reply to Post 15
|
||||
|
||||
2015 did some custom CPU-side CRT shader tools for high resolution physical paper prints for someone's book. Wasn't trying to be physically accurate, but rather provide the feeling you'd get from a CRT in paper print form
|
||||
|
||||
|
||||
|
||||
## Post 17 (2023-05-04 17:03:07) — reply to Post 16
|
||||
|
||||
@NOTimothyLottes This is when I learned about your work and found it incredible. Nowadays with 4K / 8K, people are really craving for a very good shader/model to emulate as close as possible the CRT feel for arcade / old console etc. (I am serious)
|
||||
|
||||
## Post 18 (2024-02-12 23:15:30) — reply to Post 15
|
||||
|
||||
@NOTimothyLottes Have you ever blogged details about the syntax of your Forth tools?
|
||||
|
||||
## Post 19 (2024-02-13 00:22:53) — reply to Post 18
|
||||
|
||||
@hotmultimedia Once upon a time. Not up any more. Lots of color-forth inspired variations. Designed to make x86 code generation easy. Last language effort for RDNA2 code gen wasn't forth-like though, focused on GPU compiling it's own code in parallel on the GPU (SIMD).
|
||||
@@ -0,0 +1,361 @@
|
||||
{
|
||||
"root_post_id": "1757198624818168210",
|
||||
"posts": [
|
||||
{
|
||||
"post_id": "1653556990302470151",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "Found some of my old blogs: thread looking back an image at a time. 2007, ellipsoid l-system sprites with temporal feedback, yes temporal reprojection in 2007.",
|
||||
"timestamp": "2023-05-03 00:28:02",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKcwX2WIAQNIjp?format=jpg&name=orig"
|
||||
],
|
||||
"reply_to_id": null,
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 7,
|
||||
"repost_count": 6,
|
||||
"like_count": 53,
|
||||
"view_count": 14446
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653558383415271424",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2007 particle based fluid sim mixed with spring model, this was 2D to work out ideas for later 3D implementation, rastered particle properties along particle paths into mip tree to enable high-speed collision",
|
||||
"timestamp": "2023-05-03 00:33:34",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKd2pZWcAArMRq?format=jpg&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653556990302470151",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 4,
|
||||
"view_count": 722
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653559205360553985",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2007 crazy custom language and live editor for x86-64, color forth is like a virus of the mind, no going back once the understanding is there, and the evolutions of those ideas are ground breaking",
|
||||
"timestamp": "2023-05-03 00:36:50",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKe2FbWcAITJU9?format=jpg&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653558383415271424",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 2,
|
||||
"repost_count": 0,
|
||||
"like_count": 10,
|
||||
"view_count": 746
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653561084429713408",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2010 digital photography, was compositing exposure brackets to get 16-bit tiffs with good dynamic range and did my own development tools (which had some major flaws I'd only learn about much later in life)",
|
||||
"timestamp": "2023-05-03 00:44:18",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKfd5DXwAIzmaH?format=jpg&name=orig",
|
||||
"https://pbs.twimg.com/media/FvKgFuqWcAA3uQc?format=jpg&name=orig",
|
||||
"https://pbs.twimg.com/media/FvKgJSwWIAIUavB?format=jpg&name=orig",
|
||||
"https://pbs.twimg.com/media/FvKgU0NXoAAaXK4?format=jpg&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653559205360553985",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 2,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 695
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653563093106974728",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2011 Was using temporal reprojection in my own engines for a while. Posted TSSAA source, an early TAA using viewport jitter for tri raster. I was mostly convinved this was a tech dead end due to the challenges of thin flicker. Decade+ later, and this story has a lot more to it ..",
|
||||
"timestamp": "2023-05-03 00:52:17",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKhxrpXgAEmVaS?format=png&name=orig",
|
||||
"https://pbs.twimg.com/media/FvKhzBoXgAABF7I?format=png&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653561084429713408",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 2,
|
||||
"view_count": 645
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653564005888274432",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2011 A simple GIF showing the challenges of {perceptual vs linear} filtering with temporal aliasing high frequency content with {small to large kernel windows}. No negative lobes here.",
|
||||
"timestamp": "2023-05-03 00:55:55",
|
||||
"media_urls": [
|
||||
"https://video.twimg.com/tweet_video/FvKi92TWcAA19un.mp4"
|
||||
],
|
||||
"reply_to_id": "1653563093106974728",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 2,
|
||||
"view_count": 588
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653564728440942592",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2014 simple anti-aliased point based CS rendering engine with fisheye output using stocastic/stratified visiblity and GPU-side scene graph",
|
||||
"timestamp": "2023-05-03 00:58:47",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKj76BXwAEDBNO?format=png&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653564005888274432",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 3,
|
||||
"view_count": 581
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653565160605270016",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2014 simple non-temporal spatial AA using rotated viewport to provide a good sample distribution with standard non-AA triangle rendering",
|
||||
"timestamp": "2023-05-03 01:00:30",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKkiT4WAAU2Z3a?format=jpg&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653564728440942592",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 592
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653566080235761669",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2014 monochrome 120 hz laptop ray marcher desiged for VR, fixed cost trace, leaving scene with holes, temporal holefilling reprojection, faked 'lighting' using hierarchical + directional distance probing in lower LOD scene representations, was amazing to just fly around in there",
|
||||
"timestamp": "2023-05-03 01:04:09",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKk3NoXwAApx-w?format=png&name=orig",
|
||||
"https://pbs.twimg.com/media/FvKk42ZXoAInjtk?format=png&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653565160605270016",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 6,
|
||||
"view_count": 636
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653566551818207232",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2014 was amazing how nice it was possible to get the geometry even when tracing only 1/8 the screen pixels per frame, stratified sample pattern really does wonders to neighborhood stability, no flicker at all",
|
||||
"timestamp": "2023-05-03 01:06:02",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKltaSWcAMg5ke?format=jpg&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653566080235761669",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 587
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653566948897062914",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2014 still distracted by custom code generation and languages, running stuff on CRTs with custom live editors (color forth like), generating binaries with ELF header generation inside the code itself",
|
||||
"timestamp": "2023-05-03 01:07:37",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKmJ1aXgAI9Qt9?format=png&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653566551818207232",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 3,
|
||||
"view_count": 619
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653567451315986432",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2015 some more images inside the live fractal tracer",
|
||||
"timestamp": "2023-05-03 01:09:36",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKmsMfWAAIl9g1?format=jpg&name=orig",
|
||||
"https://pbs.twimg.com/media/FvKmtisXsAIC4v-?format=jpg&name=orig",
|
||||
"https://pbs.twimg.com/media/FvKmu0XXsAEP-PW?format=jpg&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653566948897062914",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 5,
|
||||
"view_count": 659
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653568279271682052",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2015 crazy mini x86 OS project (bootloader source below), using codeless binary editor with annotation. Everyone should try generating binaries with no assembler, learn alot in the process",
|
||||
"timestamp": "2023-05-03 01:12:54",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKnKcEXoAA-pjq?format=png&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653567451315986432",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 6,
|
||||
"view_count": 911
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653569285048918018",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2015 live realtime 120+ hz on laptop tracer now with spatial/temporal reconstruction on 1 shaded/sample per ray for 'fake' lit fog, did a lot of z based logic in the spatial/temporal reconstruction, realized later I was relying on the 120 Hz to hide perceptual artifacts",
|
||||
"timestamp": "2023-05-03 01:16:54",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKoJVpWwAA9hUQ?format=jpg&name=orig",
|
||||
"https://pbs.twimg.com/media/FvKoKkEX0AAmyRq?format=jpg&name=orig",
|
||||
"https://pbs.twimg.com/media/FvKoL0ZXoAAkwWj?format=jpg&name=orig",
|
||||
"https://pbs.twimg.com/media/FvKoNDuXgAAbXBM?format=jpg&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653568279271682052",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 2,
|
||||
"repost_count": 0,
|
||||
"like_count": 5,
|
||||
"view_count": 950
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653570073288757253",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2015 still doing the crazy custom languages and tiny OS projects. I feel like there is a real loss in computer evolution, imagine a machine with C64 simplicity that evolved to have a massively parallel vector machine. It would be amazing. It would be approachable by one person.",
|
||||
"timestamp": "2023-05-03 01:20:01",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKotj4XwAEO-pU?format=png&name=orig",
|
||||
"https://pbs.twimg.com/media/FvKovAaWcAADk-H?format=png&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653569285048918018",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 2,
|
||||
"repost_count": 0,
|
||||
"like_count": 8,
|
||||
"view_count": 1128
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1653570742762479620",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "2015 did some custom CPU-side CRT shader tools for high resolution physical paper prints for someone's book. Wasn't trying to be physically accurate, but rather provide the feeling you'd get from a CRT in paper print form",
|
||||
"timestamp": "2023-05-03 01:22:41",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/FvKplmbXgAISMMM?format=jpg&name=orig"
|
||||
],
|
||||
"reply_to_id": "1653570073288757253",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 11,
|
||||
"view_count": 1168
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1654169796328423450",
|
||||
"author": "Laxer3A",
|
||||
"handle": "Laxer3A",
|
||||
"text": "@NOTimothyLottes This is when I learned about your work and found it incredible. Nowadays with 4K / 8K, people are really craving for a very good shader/model to emulate as close as possible the CRT feel for arcade / old console etc. (I am serious)",
|
||||
"timestamp": "2023-05-04 17:03:07",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1653570742762479620",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 0,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 53
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1757181664126444015",
|
||||
"author": "Alan Olivaw",
|
||||
"handle": "hotmultimedia",
|
||||
"text": "@NOTimothyLottes Have you ever blogged details about the syntax of your Forth tools?",
|
||||
"timestamp": "2024-02-12 23:15:30",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1653570073288757253",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 0,
|
||||
"view_count": 45
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1757198624818168210",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@hotmultimedia Once upon a time. Not up any more. Lots of color-forth inspired variations. Designed to make x86 code generation easy. Last language effort for RDNA2 code gen wasn't forth-like though, focused on GPU compiling it's own code in parallel on the GPU (SIMD).",
|
||||
"timestamp": "2024-02-13 00:22:53",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1757181664126444015",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 0,
|
||||
"repost_count": 0,
|
||||
"like_count": 0,
|
||||
"view_count": 71
|
||||
}
|
||||
}
|
||||
],
|
||||
"source_url": "https://x.com/NOTimothyLottes/status/1757198624818168210",
|
||||
"source_urls": [
|
||||
"https://x.com/NOTimothyLottes/status/1757198624818168210",
|
||||
"https://x.com/NOTimothyLottes/status/1653570742762479620"
|
||||
],
|
||||
"member_targets": [
|
||||
"1653570742762479620",
|
||||
"1757198624818168210"
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 91 KiB |
@@ -0,0 +1,127 @@
|
||||
---
|
||||
title: "@onatt0 [3] I got side tracked by building a language that could be assembled fr"
|
||||
author: "NOTimothyLottes"
|
||||
handle: "@NOTimothyLottes"
|
||||
post_url: "https://x.com/NOTimothyLottes/status/1917646466417381426"
|
||||
post_id: "1917646466417381426"
|
||||
timestamp: "2025-04-30 18:25:20"
|
||||
post_count: 22
|
||||
reply_count: 2
|
||||
repost_count: 0
|
||||
like_count: 1
|
||||
view_count: 125
|
||||
---
|
||||
|
||||
# @NOTimothyLottes — @onatt0 [3] I got side tracked by building a language that could be assembled fr
|
||||
|
||||
## Post 1 (2025-04-30 16:13:33)
|
||||
|
||||
Forgot to mention in the heat of presentation, the initial textual language was heavily influenced by one of your past forth-like languages.
|
||||
Though I've built upon that foundation, I would have taken many wrong turns without your guidance, thank you so much.
|
||||
|
||||
## Post 2 (2025-04-30 18:10:43) — reply to Post 1
|
||||
|
||||
@onatt0 Related thoughts/
|
||||
[0] Having a low upper bound on the maximum complexity allowed in a program enables so much simplification. One can always move complexity into data, while keeping tight codebases.
|
||||
|
||||
## Post 3 (2025-04-30 18:19:07) — reply to Post 2
|
||||
|
||||
@onatt0 [1] Seems like you group symbols into pages where each page can have a string (shared with all symbols in the page), which when pared with limited fixed maximum symbol string size, is an elegant way of effectively supporting larger naming [I'll probably steal that idea next time]
|
||||
|
||||
## Post 4 (2025-04-30 18:22:55) — reply to Post 3
|
||||
|
||||
@onatt0 [2] I'm also a big fan of how you used 16:9 aspect to auto render all the debug info, symbol tables, disassembly, etc, alongside the source. I think many people are probably lost in the speed at which you can manipulate and test ideas while working on the source
|
||||
|
||||
## Post 5 (2025-04-30 18:25:20) — reply to Post 4
|
||||
|
||||
@onatt0 [3] I got side tracked by building a language that could be assembled from on the GPU in SIMD. However now I ask myself if that is just adding "complexity", because if programs are bounded in size, why not just focus on CPU non-parallel nested factoring (aka the forth-like way)
|
||||
|
||||
## Post 6 (2025-04-30 18:27:04) — reply to Post 3
|
||||
|
||||
@NOTimothyLottes oh dang realized i forgot to mention that part in the talk:)
|
||||
symbol name encoding for this iteration is 63bits(7bits*9chars) + 1bit extend(which highlights the word)
|
||||
extension can be at either end,
|
||||
say VFRAMEADDR AFRAMEADDR VPLANEADDR
|
||||
or VPACK.RING VPACK.HEAD VPACK.TAIL
|
||||
|
||||
|
||||
|
||||
## Post 7 (2025-04-30 18:35:00) — reply to Post 5
|
||||
|
||||
@onatt0 [4] 2-item data stack is an interesting compromise. Something I never considered. I left off ripping out the data stack completely.
|
||||
|
||||
## Post 8 (2025-04-30 18:40:32) — reply to Post 7
|
||||
|
||||
@onatt0 [5] Can do this instead,
|
||||
a. Track a "top" register (number)
|
||||
b. Use symbols to override top register
|
||||
c. Have push (store) just advance top to next reg (in circular queue)
|
||||
Gets to easy unnamed arguments
|
||||
|
||||
## Post 9 (2025-04-30 18:45:00) — reply to Post 5
|
||||
|
||||
@NOTimothyLottes IMO code compilation is inherently sequential+ "execution contextual", dynamic logic execution to get the binary as the essence vs a static lang
|
||||
|
||||
codegen with execution on GPU could work as 2-items(vreg)/stack and 32K cells per lane for AI to map code to data instead of weights?
|
||||
|
||||
## Post 10 (2025-04-30 18:45:38) — reply to Post 8
|
||||
|
||||
@onatt0 [6] You mentioned VK is most "form filling" which I think is an accurate description. For most "C" like APIs I like to just lay out all the arguments in memory like a tape drive in the order that functions get called and source that tape at runtime for the calls ...
|
||||
|
||||
## Post 11 (2025-04-30 18:47:28) — reply to Post 10
|
||||
|
||||
@onatt0 [7] They key concept here is that "common" arguments like the device are pushed onto the tape using store duplication when they are known (after device creation). So it's preemptive scatter, so later at call time there is no argument gather.
|
||||
|
||||
## Post 12 (2025-04-30 18:49:40) — reply to Post 11
|
||||
|
||||
@onatt0 [8] Likely the majority of C/C++/OOP/bloatware is just shuffling data around in argument gather to support the concept of data stacks on HW that has no physical data stack.
|
||||
|
||||
## Post 13 (2025-04-30 18:52:21) — reply to Post 12
|
||||
|
||||
@onatt0 [9] Could just pre-layout call arguments in order of usage, leverage data compression at init-time to unpack into memory before run-time. No more code to shuffle arguments, or set registers to immediates, etc. Just a common (cached) pre-call to consume args from the "tape"
|
||||
|
||||
## Post 14 (2025-04-30 18:53:48) — reply to Post 12
|
||||
|
||||
@NOTimothyLottes holy truthnuke - and people think C is the optimal state of possible runtimes when it's a very limited runtime to have state mixed call/data-stacks
|
||||
|
||||
not only you have to keep the whole stack around with replicated state, it limits to serialized execution instead of parallel
|
||||
|
||||
## Post 15 (2025-04-30 18:55:33) — reply to Post 13
|
||||
|
||||
@NOTimothyLottes simply brilliant!
|
||||
|
||||
## Post 16 (2025-04-30 18:59:14) — reply to Post 9
|
||||
|
||||
@onatt0 So for x86 I've many times written code gen with extra nop-prefix padding to fix all instructions to a multiple of known 32-bits. Which means you know in advance sizing of everything, then you can easily do parallel code generation. So need not necessarily all be serial
|
||||
|
||||
## Post 17 (2025-04-30 19:03:11) — reply to Post 16
|
||||
|
||||
@onatt0 However I'm often building then instantly using code while generating a baked binary to use later, so that regard much is inherently serially dependent. This bootstrapping technique is perhaps one the great things to learn from things like color forth IMO.
|
||||
|
||||
## Post 18 (2025-04-30 19:04:57) — reply to Post 14
|
||||
|
||||
@onatt0 I laugh when people say C is like assembly, they are missing what we actually did in assembly back then, which was all registers and globals and gotos, no stacks. It's radically different than good assembly.
|
||||
|
||||
## Post 19 (2025-04-30 19:05:05) — reply to Post 6
|
||||
|
||||
@NOTimothyLottes a much better encoding is possible
|
||||
next iteration is going to be
|
||||
|
||||
16bits*4words if you need multiple words(last word=15bits)
|
||||
7bits*9chars if its a unique/short name(to not clutter the 16-bit dictionary-space)
|
||||
|
||||
64K*4 words should be enough for most cases
|
||||
|
||||
## Post 20 (2025-04-30 19:10:53) — reply to Post 19
|
||||
|
||||
@onatt0 It would be interesting to size stuff so the limits align with instruction cache size limits. Build something where the expectation is a hit in the I$ once it's warm after the context switch and move everything else to data complexity.
|
||||
|
||||
## Post 21 (2025-04-30 19:13:59) — reply to Post 20
|
||||
|
||||
@onatt0 Same could apply for non-bulk data, ie the stuff you'd possibly want human readable annotation on. Make it sized to practical L2$ limits, or fix it to at most the shared L3$.
|
||||
|
||||
## Post 22 (2025-04-30 19:15:11) — reply to Post 18
|
||||
|
||||
@NOTimothyLottes when C became *the* execution model, it restricted all future hardware, HW gets built with how the C compiler will compile to it instead of what's ultimately a good design and a malleable macro-lang to map to HW
|
||||
|
||||
ofc a lot of people want portability so we went the boring route
|
||||
@@ -0,0 +1,372 @@
|
||||
{
|
||||
"root_post_id": "1917646466417381426",
|
||||
"posts": [
|
||||
{
|
||||
"post_id": "1917613300902146419",
|
||||
"author": "on@☦️",
|
||||
"handle": "onatt0",
|
||||
"text": "Forgot to mention in the heat of presentation, the initial textual language was heavily influenced by one of your past forth-like languages. \nThough I've built upon that foundation, I would have taken many wrong turns without your guidance, thank you so much.",
|
||||
"timestamp": "2025-04-30 16:13:33",
|
||||
"media_urls": [],
|
||||
"reply_to_id": null,
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 5,
|
||||
"view_count": 308
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917642786804785230",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 Related thoughts/\n[0] Having a low upper bound on the maximum complexity allowed in a program enables so much simplification. One can always move complexity into data, while keeping tight codebases.",
|
||||
"timestamp": "2025-04-30 18:10:43",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917613300902146419",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 2,
|
||||
"view_count": 112
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917644904055910502",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 [1] Seems like you group symbols into pages where each page can have a string (shared with all symbols in the page), which when pared with limited fixed maximum symbol string size, is an elegant way of effectively supporting larger naming [I'll probably steal that idea next time]",
|
||||
"timestamp": "2025-04-30 18:19:07",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917642786804785230",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 2,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 93
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917645859791200562",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 [2] I'm also a big fan of how you used 16:9 aspect to auto render all the debug info, symbol tables, disassembly, etc, alongside the source. I think many people are probably lost in the speed at which you can manipulate and test ideas while working on the source",
|
||||
"timestamp": "2025-04-30 18:22:55",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917644904055910502",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 88
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917646466417381426",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 [3] I got side tracked by building a language that could be assembled from on the GPU in SIMD. However now I ask myself if that is just adding \"complexity\", because if programs are bounded in size, why not just focus on CPU non-parallel nested factoring (aka the forth-like way)",
|
||||
"timestamp": "2025-04-30 18:25:20",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917645859791200562",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 2,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 125
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917646904277754258",
|
||||
"author": "on@☦️",
|
||||
"handle": "onatt0",
|
||||
"text": "@NOTimothyLottes oh dang realized i forgot to mention that part in the talk:) \nsymbol name encoding for this iteration is 63bits(7bits*9chars) + 1bit extend(which highlights the word) \nextension can be at either end, \nsay VFRAMEADDR AFRAMEADDR VPLANEADDR\nor VPACK.RING VPACK.HEAD VPACK.TAIL",
|
||||
"timestamp": "2025-04-30 18:27:04",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/GpzZmVBW8AALxll?format=png&name=orig"
|
||||
],
|
||||
"reply_to_id": "1917644904055910502",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 27
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917648900556558768",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 [4] 2-item data stack is an interesting compromise. Something I never considered. I left off ripping out the data stack completely.",
|
||||
"timestamp": "2025-04-30 18:35:00",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917646466417381426",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 87
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917650289978454504",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 [5] Can do this instead,\na. Track a \"top\" register (number)\nb. Use symbols to override top register\nc. Have push (store) just advance top to next reg (in circular queue)\nGets to easy unnamed arguments",
|
||||
"timestamp": "2025-04-30 18:40:32",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917648900556558768",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 2,
|
||||
"repost_count": 0,
|
||||
"like_count": 3,
|
||||
"view_count": 108
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917651417487036446",
|
||||
"author": "on@☦️",
|
||||
"handle": "onatt0",
|
||||
"text": "@NOTimothyLottes IMO code compilation is inherently sequential+ \"execution contextual\", dynamic logic execution to get the binary as the essence vs a static lang\n\ncodegen with execution on GPU could work as 2-items(vreg)/stack and 32K cells per lane for AI to map code to data instead of weights?",
|
||||
"timestamp": "2025-04-30 18:45:00",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917646466417381426",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 0,
|
||||
"view_count": 21
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917651574354030636",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 [6] You mentioned VK is most \"form filling\" which I think is an accurate description. For most \"C\" like APIs I like to just lay out all the arguments in memory like a tape drive in the order that functions get called and source that tape at runtime for the calls ...",
|
||||
"timestamp": "2025-04-30 18:45:38",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917650289978454504",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 98
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917652037078065160",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 [7] They key concept here is that \"common\" arguments like the device are pushed onto the tape using store duplication when they are known (after device creation). So it's preemptive scatter, so later at call time there is no argument gather.",
|
||||
"timestamp": "2025-04-30 18:47:28",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917651574354030636",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 2,
|
||||
"view_count": 110
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917652589358858332",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 [8] Likely the majority of C/C++/OOP/bloatware is just shuffling data around in argument gather to support the concept of data stacks on HW that has no physical data stack.",
|
||||
"timestamp": "2025-04-30 18:49:40",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917652037078065160",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 2,
|
||||
"repost_count": 1,
|
||||
"like_count": 3,
|
||||
"view_count": 209
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917653265329594372",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 [9] Could just pre-layout call arguments in order of usage, leverage data compression at init-time to unpack into memory before run-time. No more code to shuffle arguments, or set registers to immediates, etc. Just a common (cached) pre-call to consume args from the \"tape\"",
|
||||
"timestamp": "2025-04-30 18:52:21",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917652589358858332",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 2,
|
||||
"repost_count": 0,
|
||||
"like_count": 4,
|
||||
"view_count": 135
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917653631651729876",
|
||||
"author": "on@☦️",
|
||||
"handle": "onatt0",
|
||||
"text": "@NOTimothyLottes holy truthnuke - and people think C is the optimal state of possible runtimes when it's a very limited runtime to have state mixed call/data-stacks\n\nnot only you have to keep the whole stack around with replicated state, it limits to serialized execution instead of parallel",
|
||||
"timestamp": "2025-04-30 18:53:48",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917652589358858332",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 0,
|
||||
"view_count": 80
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917654071323857138",
|
||||
"author": "on@☦️",
|
||||
"handle": "onatt0",
|
||||
"text": "@NOTimothyLottes simply brilliant!",
|
||||
"timestamp": "2025-04-30 18:55:33",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917653265329594372",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 0,
|
||||
"repost_count": 0,
|
||||
"like_count": 0,
|
||||
"view_count": 48
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917654997895983255",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 So for x86 I've many times written code gen with extra nop-prefix padding to fix all instructions to a multiple of known 32-bits. Which means you know in advance sizing of everything, then you can easily do parallel code generation. So need not necessarily all be serial",
|
||||
"timestamp": "2025-04-30 18:59:14",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917651417487036446",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 43
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917655990792560843",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 However I'm often building then instantly using code while generating a baked binary to use later, so that regard much is inherently serially dependent. This bootstrapping technique is perhaps one the great things to learn from things like color forth IMO.",
|
||||
"timestamp": "2025-04-30 19:03:11",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917654997895983255",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 0,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 58
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917656437473399108",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 I laugh when people say C is like assembly, they are missing what we actually did in assembly back then, which was all registers and globals and gotos, no stacks. It's radically different than good assembly.",
|
||||
"timestamp": "2025-04-30 19:04:57",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917653631651729876",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 1,
|
||||
"like_count": 3,
|
||||
"view_count": 156
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917656470444790255",
|
||||
"author": "on@☦️",
|
||||
"handle": "onatt0",
|
||||
"text": "@NOTimothyLottes a much better encoding is possible\nnext iteration is going to be\n\n16bits*4words if you need multiple words(last word=15bits)\n7bits*9chars if its a unique/short name(to not clutter the 16-bit dictionary-space)\n\n64K*4 words should be enough for most cases",
|
||||
"timestamp": "2025-04-30 19:05:05",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917646904277754258",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 0,
|
||||
"view_count": 20
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917657931375313017",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 It would be interesting to size stuff so the limits align with instruction cache size limits. Build something where the expectation is a hit in the I$ once it's warm after the context switch and move everything else to data complexity.",
|
||||
"timestamp": "2025-04-30 19:10:53",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917656470444790255",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 34
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917658709192306788",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@onatt0 Same could apply for non-bulk data, ie the stuff you'd possibly want human readable annotation on. Make it sized to practical L2$ limits, or fix it to at most the shared L3$.",
|
||||
"timestamp": "2025-04-30 19:13:59",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917657931375313017",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 0,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 48
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1917659010024587634",
|
||||
"author": "on@☦️",
|
||||
"handle": "onatt0",
|
||||
"text": "@NOTimothyLottes when C became *the* execution model, it restricted all future hardware, HW gets built with how the C compiler will compile to it instead of what's ultimately a good design and a malleable macro-lang to map to HW\n\nofc a lot of people want portability so we went the boring route",
|
||||
"timestamp": "2025-04-30 19:15:11",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1917656437473399108",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 40
|
||||
}
|
||||
}
|
||||
],
|
||||
"source_url": "https://x.com/NOTimothyLottes/status/1917646466417381426",
|
||||
"source_urls": [
|
||||
"https://x.com/NOTimothyLottes/status/1917646466417381426",
|
||||
"https://x.com/NOTimothyLottes/status/1917644904055910502",
|
||||
"https://x.com/NOTimothyLottes/status/1917645859791200562",
|
||||
"https://x.com/NOTimothyLottes/status/1917642786804785230"
|
||||
],
|
||||
"member_targets": [
|
||||
"1917642786804785230",
|
||||
"1917644904055910502",
|
||||
"1917645859791200562",
|
||||
"1917646466417381426"
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.8 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.0 KiB |
@@ -0,0 +1,61 @@
|
||||
---
|
||||
title: "@NOTimothyLottes i'm not familiar with your problem."
|
||||
author: "/i:'mɪər/"
|
||||
handle: "@VPCOMPRESSB"
|
||||
post_url: "https://x.com/VPCOMPRESSB/status/1987744335333622188?s=20"
|
||||
post_id: "1987744335333622188"
|
||||
timestamp: "2025-11-10 04:49:14"
|
||||
post_count: 7
|
||||
reply_count: 0
|
||||
repost_count: 0
|
||||
like_count: 0
|
||||
view_count: 43
|
||||
---
|
||||
|
||||
# @VPCOMPRESSB — @NOTimothyLottes i'm not familiar with your problem.
|
||||
|
||||
## Post 1 (2025-11-07 21:04:00)
|
||||
|
||||
Will be switching back to SteamDeck based Linux dev for home after next week. Looking forward to it. Old setup here. Would swap CRTs often. Bonus I'll be able to test the AnyBeam laser projector at custom resolutions too.
|
||||
|
||||
|
||||
|
||||
## Post 2 (2025-11-07 21:13:13) — reply to Post 1
|
||||
|
||||
Will be returning to all custom color-forth-inspired macro assembly coding. I've done a huge amount of exploration in custom minimal languages, wanted to put my best out there in a public domain project.
|
||||
|
||||
## Post 3 (2025-11-07 22:05:43) — reply to Post 2
|
||||
|
||||
i'm developing a simple Forth interpreter as a project for school.
|
||||
|
||||
no tokens (in the conventional sense). no syntax tree. i could implement strings, but i want to omit them.
|
||||
|
||||
a program would just be a kernel that's invoked per element of some buffer.
|
||||
|
||||
i must write it in Java, which makes development more difficult and irritating IMO.
|
||||
|
||||
## Post 4 (2025-11-10 02:56:27) — reply to Post 3
|
||||
|
||||
@NOTimothyLottes i also only want 2-character words.
|
||||
including numbers.
|
||||
|
||||
i only use numbers of the form: `N << M` or `N * MB`.
|
||||
|
||||
i also should default to hexadecimals.
|
||||
|
||||
## Post 5 (2025-11-10 03:04:34) — reply to Post 4
|
||||
|
||||
@NOTimothyLottes there are 95 printable characters on the standard USA keyboard.
|
||||
95^2 = 9025 different typable words.
|
||||
that's far more than the 3383 instructions in the Zen 4.
|
||||
and far far more than i'll ever need.
|
||||
|
||||
|
||||
|
||||
## Post 6 (2025-11-10 04:05:34) — reply to Post 5
|
||||
|
||||
@VPCOMPRESSB Logic on max word count is probably sound. But I've never been able to get all symbol length to 2 chars.
|
||||
|
||||
## Post 7 (2025-11-10 04:49:14) — reply to Post 6
|
||||
|
||||
@NOTimothyLottes i'm not familiar with your problem.
|
||||
@@ -0,0 +1,128 @@
|
||||
{
|
||||
"root_post_id": "1987744335333622188",
|
||||
"posts": [
|
||||
{
|
||||
"post_id": "1986902480609767497",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "Will be switching back to SteamDeck based Linux dev for home after next week. Looking forward to it. Old setup here. Would swap CRTs often. Bonus I'll be able to test the AnyBeam laser projector at custom resolutions too.",
|
||||
"timestamp": "2025-11-07 21:04:00",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/G5LkqJtWMAAuS6S?format=jpg&name=orig"
|
||||
],
|
||||
"reply_to_id": null,
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 1,
|
||||
"like_count": 19,
|
||||
"view_count": 2400
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1986904798554099822",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "Will be returning to all custom color-forth-inspired macro assembly coding. I've done a huge amount of exploration in custom minimal languages, wanted to put my best out there in a public domain project.",
|
||||
"timestamp": "2025-11-07 21:13:13",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1986902480609767497",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 2,
|
||||
"repost_count": 0,
|
||||
"like_count": 15,
|
||||
"view_count": 1227
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1986918008762286143",
|
||||
"author": "/i:'mɪər/",
|
||||
"handle": "VPCOMPRESSB",
|
||||
"text": "i'm developing a simple Forth interpreter as a project for school.\n\nno tokens (in the conventional sense). no syntax tree. i could implement strings, but i want to omit them.\n\na program would just be a kernel that's invoked per element of some buffer.\n\ni must write it in Java, which makes development more difficult and irritating IMO.",
|
||||
"timestamp": "2025-11-07 22:05:43",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1986904798554099822",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 3,
|
||||
"view_count": 160
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1987715952679510516",
|
||||
"author": "/i:'mɪər/",
|
||||
"handle": "VPCOMPRESSB",
|
||||
"text": "@NOTimothyLottes i also only want 2-character words.\nincluding numbers.\n\ni only use numbers of the form: `N << M` or `N * MB`.\n\ni also should default to hexadecimals.",
|
||||
"timestamp": "2025-11-10 02:56:27",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1986918008762286143",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 0,
|
||||
"view_count": 39
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1987717996249260225",
|
||||
"author": "/i:'mɪər/",
|
||||
"handle": "VPCOMPRESSB",
|
||||
"text": "@NOTimothyLottes there are 95 printable characters on the standard USA keyboard.\n95^2 = 9025 different typable words.\nthat's far more than the 3383 instructions in the Zen 4.\nand far far more than i'll ever need.",
|
||||
"timestamp": "2025-11-10 03:04:34",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/G5XKSCbXAAEJsfl?format=png&name=orig"
|
||||
],
|
||||
"reply_to_id": "1987715952679510516",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 1,
|
||||
"view_count": 79
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1987733345628013014",
|
||||
"author": "NOTimothyLottes",
|
||||
"handle": "NOTimothyLottes",
|
||||
"text": "@VPCOMPRESSB Logic on max word count is probably sound. But I've never been able to get all symbol length to 2 chars.",
|
||||
"timestamp": "2025-11-10 04:05:34",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1987717996249260225",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 1,
|
||||
"repost_count": 0,
|
||||
"like_count": 0,
|
||||
"view_count": 112
|
||||
}
|
||||
},
|
||||
{
|
||||
"post_id": "1987744335333622188",
|
||||
"author": "/i:'mɪər/",
|
||||
"handle": "VPCOMPRESSB",
|
||||
"text": "@NOTimothyLottes i'm not familiar with your problem.",
|
||||
"timestamp": "2025-11-10 04:49:14",
|
||||
"media_urls": [],
|
||||
"reply_to_id": "1987733345628013014",
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 0,
|
||||
"repost_count": 0,
|
||||
"like_count": 0,
|
||||
"view_count": 43
|
||||
}
|
||||
}
|
||||
],
|
||||
"source_url": "https://x.com/VPCOMPRESSB/status/1987744335333622188?s=20",
|
||||
"source_urls": [
|
||||
"https://x.com/VPCOMPRESSB/status/1987744335333622188?s=20"
|
||||
],
|
||||
"member_targets": [
|
||||
"1987744335333622188"
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 205 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 443 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 333 KiB |
@@ -0,0 +1,29 @@
|
||||
---
|
||||
title: "no Tokens and no ASTs. (easier because its Forth-like)."
|
||||
author: "/i:'mɪər/"
|
||||
handle: "@VPCOMPRESSB"
|
||||
post_url: "https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20"
|
||||
post_id: "1991383117571957052"
|
||||
timestamp: "2025-11-20 05:48:27"
|
||||
post_count: 1
|
||||
reply_count: 0
|
||||
repost_count: 0
|
||||
like_count: 4
|
||||
view_count: 292
|
||||
---
|
||||
|
||||
# @VPCOMPRESSB — no Tokens and no ASTs. (easier because its Forth-like).
|
||||
|
||||
## Post 1 (2025-11-20 05:48:27)
|
||||
|
||||
no Tokens and no ASTs. (easier because its Forth-like).
|
||||
|
||||
all i'm doing is transforming the source directly via byte shuffling and bit magic, and its parallelizable at a per-byte basis [theoretically, because i have yet to try that out].
|
||||
|
||||
code is awful because i'm using Java as if it's C, and it isn't optimized (plenty of optimizations that i've setup) nor cleaned.
|
||||
|
||||
the <\> represents NULL. (this was debugged with print 🙂).
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"root_post_id": "1991383117571957052",
|
||||
"posts": [
|
||||
{
|
||||
"post_id": "1991383117571957052",
|
||||
"author": "/i:'mɪər/",
|
||||
"handle": "VPCOMPRESSB",
|
||||
"text": "no Tokens and no ASTs. (easier because its Forth-like).\n\nall i'm doing is transforming the source directly via byte shuffling and bit magic, and its parallelizable at a per-byte basis [theoretically, because i have yet to try that out].\n\ncode is awful because i'm using Java as if it's C, and it isn't optimized (plenty of optimizations that i've setup) nor cleaned.\n\nthe <\\> represents NULL. (this was debugged with print 🙂).",
|
||||
"timestamp": "2025-11-20 05:48:27",
|
||||
"media_urls": [
|
||||
"https://pbs.twimg.com/media/G6LJrEgXkAA2Z26?format=png&name=orig",
|
||||
"https://pbs.twimg.com/media/G6LPXFOWAAABSVy?format=jpg&name=orig",
|
||||
"https://pbs.twimg.com/media/G6LPYhqXgAA2rvt?format=jpg&name=orig"
|
||||
],
|
||||
"reply_to_id": null,
|
||||
"quote_of_id": null,
|
||||
"metrics": {
|
||||
"reply_count": 0,
|
||||
"repost_count": 0,
|
||||
"like_count": 4,
|
||||
"view_count": 292
|
||||
}
|
||||
}
|
||||
],
|
||||
"source_url": "https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20",
|
||||
"source_urls": [
|
||||
"https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20"
|
||||
],
|
||||
"member_targets": [
|
||||
"1991383117571957052"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
[
|
||||
{
|
||||
"target": "1757198624818168210",
|
||||
"handle": "NOTimothyLottes",
|
||||
"posts": 19,
|
||||
"media": 27,
|
||||
"source_urls": [
|
||||
"https://x.com/NOTimothyLottes/status/1757198624818168210",
|
||||
"https://x.com/NOTimothyLottes/status/1653570742762479620"
|
||||
]
|
||||
},
|
||||
{
|
||||
"target": "1917646466417381426",
|
||||
"handle": "NOTimothyLottes",
|
||||
"posts": 22,
|
||||
"media": 1,
|
||||
"source_urls": [
|
||||
"https://x.com/NOTimothyLottes/status/1917646466417381426",
|
||||
"https://x.com/NOTimothyLottes/status/1917644904055910502",
|
||||
"https://x.com/NOTimothyLottes/status/1917645859791200562",
|
||||
"https://x.com/NOTimothyLottes/status/1917642786804785230"
|
||||
]
|
||||
},
|
||||
{
|
||||
"target": "1987744335333622188",
|
||||
"handle": "VPCOMPRESSB",
|
||||
"posts": 7,
|
||||
"media": 2,
|
||||
"source_urls": [
|
||||
"https://x.com/VPCOMPRESSB/status/1987744335333622188?s=20"
|
||||
]
|
||||
},
|
||||
{
|
||||
"target": "1991383117571957052",
|
||||
"handle": "VPCOMPRESSB",
|
||||
"posts": 1,
|
||||
"media": 3,
|
||||
"source_urls": [
|
||||
"https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,132 @@
|
||||
# twitter_threads — standalone Twitter/X thread extraction tooling
|
||||
|
||||
Acquire Twitter/X posts + threads + their media and emit a self-contained Markdown
|
||||
document per thread (a `thread.md` plus a sibling `media/` directory). The output is
|
||||
archivable, readable offline, and consumable by downstream analysis pipelines without
|
||||
any dependency on the live X.com service.
|
||||
|
||||
This package is the Twitter/X analog of `scripts/video_analysis/`: a small,
|
||||
dependency-light pipeline that acquires source content and emits Markdown. It is
|
||||
**standalone** — it imports nothing from `src/` or `conductor/` and can be copied to
|
||||
another repo with only `gallery-dl` as an external dependency.
|
||||
|
||||
## Modules
|
||||
|
||||
| File | Responsibility |
|
||||
|---|---|
|
||||
| `fetch_thread.py` | Acquire a post/thread into a `ThreadData`. URL path wraps `gallery-dl` (subprocess); local-HTML path uses stdlib `html.parser`. Also the pipeline entry CLI (writes `thread_data.json`). |
|
||||
| `download_media.py` | Download the images/videos/GIFs referenced by a `ThreadData` via stdlib `urllib` into `media/`. Idempotent (skips existing files). |
|
||||
| `render_markdown.py` | Emit the Markdown document (YAML front-matter + per-post sections + `./media/` links) from a `ThreadData`. |
|
||||
| `error_types.py` | Shared `Result[T]` + `ErrorInfo` (data-oriented error handling) and the `ThreadData` / `PostData` / `PostMetrics` dataclasses. |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- **Python 3.11+**
|
||||
- **`gallery-dl`** for the URL acquisition path: `pip install gallery-dl`
|
||||
- **`cookies.txt`** exported from your browser (for auth-gated content). Export via a
|
||||
browser extension (e.g. "Get cookies.txt") while logged in to x.com.
|
||||
|
||||
The script internals use **stdlib only** (`urllib`, `html.parser`, `json`, `pathlib`,
|
||||
`dataclasses`, `subprocess`). `gallery-dl` is invoked as a subprocess, never imported.
|
||||
|
||||
## Usage
|
||||
|
||||
Run each stage in order. From this repo (module form):
|
||||
|
||||
```bash
|
||||
# 1. Fetch the thread -> writes <output>/<post_id>/thread_data.json
|
||||
uv run python -m scripts.twitter_threads.fetch_thread \
|
||||
"https://x.com/NOTimothyLottes/status/1757198624818168210" \
|
||||
--output ./thread_output/ --cookies ./cookies.txt
|
||||
|
||||
# 2. Download the media -> writes <post_id>/media/<post_id>_img1.jpg etc.
|
||||
uv run python -m scripts.twitter_threads.download_media \
|
||||
--input ./thread_output/1757198624818168210/thread_data.json \
|
||||
--output ./thread_output/1757198624818168210/media/
|
||||
|
||||
# 3. Render Markdown -> writes thread.md linking ./media/<file>
|
||||
uv run python -m scripts.twitter_threads.render_markdown \
|
||||
--input ./thread_output/1757198624818168210/thread_data.json \
|
||||
--media-dir ./thread_output/1757198624818168210/media/ \
|
||||
--output ./thread_output/1757198624818168210/thread.md
|
||||
```
|
||||
|
||||
Standalone (after copying the directory to another repo — run the files directly):
|
||||
|
||||
```bash
|
||||
python fetch_thread.py "https://x.com/<user>/status/<id>" --output ./out/ --cookies ./cookies.txt
|
||||
python download_media.py --input ./out/<id>/thread_data.json --output ./out/<id>/media/
|
||||
python render_markdown.py --input ./out/<id>/thread_data.json --media-dir ./out/<id>/media/ --output ./out/<id>/thread.md
|
||||
```
|
||||
|
||||
Each module also supports `--help`.
|
||||
|
||||
### Local-HTML fallback (no network / gallery-dl broken)
|
||||
|
||||
If `gallery-dl` fails (auth wall, rate limit, X.com breakage), save the thread page
|
||||
from your browser (Ctrl+S, "Webpage, Complete") and pass the local `.html` file to
|
||||
`fetch_thread.py` instead of a URL:
|
||||
|
||||
```bash
|
||||
python fetch_thread.py ./saved_thread.html --output ./out/
|
||||
```
|
||||
|
||||
The parser reads `<article>` elements carrying `data-post-id` / `data-author` /
|
||||
`data-handle` / `data-reply-to` / `data-quote-of` / metric attributes, a
|
||||
`<time datetime>` element, a `<div data-testid="tweetText">` and `<img>`/`<source>`
|
||||
media elements.
|
||||
|
||||
## Output layout
|
||||
|
||||
```
|
||||
thread_output/
|
||||
<post_id>/
|
||||
thread_data.json # intermediate ThreadData (JSON)
|
||||
thread.md # the rendered document
|
||||
media/
|
||||
<post_id>_img1.jpg
|
||||
<post_id>_vid1.mp4
|
||||
```
|
||||
|
||||
## Output Markdown schema
|
||||
|
||||
A YAML front-matter block (author, handle, post_url, post_id, timestamp, post_count,
|
||||
reply/repost/like/view counts — `view_count: null` when unavailable), followed by an
|
||||
`# @<handle> — <title>` heading and one `## Post N (<timestamp>)` section per post (in
|
||||
chronological order, with `— reply to Post M` markers on replies). Media are linked via
|
||||
relative `./media/<filename>` paths. Quote-tweets render as a `>` blockquote linking the
|
||||
quoted post.
|
||||
|
||||
## URL normalization
|
||||
|
||||
Quote-share URLs carry a `?s=20` (or similar) suffix that is quote-share context, not
|
||||
part of the post identity. `fetch_thread.py` strips the query string + fragment before
|
||||
acquisition.
|
||||
|
||||
## Acquisition strategies (and their tradeoffs)
|
||||
|
||||
- **Strategy A — `gallery-dl` (default, implemented).** Mature, actively maintained;
|
||||
handles auth via `--cookies cookies.txt`. Wrapped as a subprocess.
|
||||
- **Strategy B — `snscrape` (legacy fallback, not bundled).** Was the standard pre-2023
|
||||
tool; increasingly broken against X.com. Usable for some older content; not a
|
||||
dependency here.
|
||||
- **Strategy C — local HTML parse (implemented, no-network fallback).** Save the page in
|
||||
the browser; parse it with stdlib `html.parser`. Most resilient when `gallery-dl`
|
||||
breaks.
|
||||
- **Strategy D — `tweepy` / Twitter API v2 (paid, not implemented).** If you have
|
||||
developer credentials, swap the acquisition backend in `fetch_thread_from_url`. Not
|
||||
the default because it requires paid API access.
|
||||
|
||||
## Copy to another repo
|
||||
|
||||
Copy the entire `scripts/twitter_threads/` directory. The only external dependency is
|
||||
`gallery-dl` (`pip install gallery-dl`). The modules use a dual-import so they run both
|
||||
as a package (`python -m ...`) and as bare scripts (`python fetch_thread.py ...`). There
|
||||
are **no imports from `src/`, `conductor/`, or `scripts.video_analysis`** — the package
|
||||
is self-contained.
|
||||
|
||||
## Conventions
|
||||
|
||||
Per `conductor/code_styleguides/python.md`: 1-space indentation, type hints, no inline
|
||||
comments. Per `conductor/code_styleguides/error_handling.md`: functions return
|
||||
`Result[T]` with `ErrorInfo` as data (no exceptions across the public API).
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Twitter/X thread extraction reusable tooling.
|
||||
|
||||
Standalone scripts that acquire posts, threads, and media from X (Twitter)
|
||||
and emit self-contained Markdown documents.
|
||||
|
||||
Scripts in this namespace:
|
||||
- fetch_thread.py: acquire a post/thread (gallery-dl subprocess for URLs; html.parser for local HTML)
|
||||
- download_media.py: download referenced media via stdlib urllib
|
||||
- render_markdown.py: emit the Markdown document + media links
|
||||
- error_types.py: shared Result-style ErrorInfo + the ThreadData/PostData/PostMetrics dataclasses
|
||||
|
||||
Standalone: no imports from src/ or conductor/; copy-pasteable to another repo with only gallery-dl as an external dep.
|
||||
|
||||
Per conductor/code_styleguides/python.md, 1-space indent + type hints + no comments in implementation code.
|
||||
"""
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Download a Twitter/X thread's media via gallery-dl (handles auth, 403, and video/GIF -> mp4)."""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
try:
|
||||
from scripts.twitter_threads.error_types import ErrorInfo, make_error, Result, ThreadData, thread_from_dict
|
||||
except ImportError:
|
||||
from error_types import ErrorInfo, make_error, Result, ThreadData, thread_from_dict
|
||||
|
||||
|
||||
|
||||
def download_media(thread: ThreadData, output_dir: Path, cookies_path: Path | None = None) -> Result[list[Path]]:
|
||||
try:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
return Result.err(make_error("WriteError", "download_media", str(e)))
|
||||
args = ["gallery-dl", "-o", "text-tweets=true", "-o", "conversations=true", "-D", str(output_dir)]
|
||||
if cookies_path is not None:
|
||||
args += ["--cookies", str(cookies_path)]
|
||||
args.append(thread.source_url)
|
||||
try:
|
||||
completed = subprocess.run(args, capture_output=True, text=True)
|
||||
except OSError as e:
|
||||
return Result.err(make_error("GalleryDlError", "download_media", str(e)))
|
||||
if completed.returncode != 0:
|
||||
return Result.err(make_error("GalleryDlError", "download_media", completed.stderr[:500]))
|
||||
post_ids = {p.post_id for p in thread.posts}
|
||||
files = [f for f in output_dir.rglob("*") if f.is_file() and f.name.split("_")[0] in post_ids]
|
||||
return Result.ok(sorted(files, key=lambda f: f.name))
|
||||
|
||||
def media_files_for_post(output_dir: Path, post_id: str) -> list[str]:
|
||||
return sorted(f.name for f in output_dir.glob(f"{post_id}_*") if f.is_file())
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Download media for a Twitter/X thread_data.json via gallery-dl.")
|
||||
parser.add_argument("--input", type=Path, required=True, help="thread_data.json path")
|
||||
parser.add_argument("--output", type=Path, required=True, help="media output directory")
|
||||
parser.add_argument("--cookies", type=Path, default=None, help="cookies.txt for gallery-dl auth")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
raw = args.input.read_text(encoding="utf-8")
|
||||
except OSError as e:
|
||||
sys.stderr.write(f"error: ReadError: {e}\n")
|
||||
return 1
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
sys.stderr.write(f"error: JsonParseError: {e}\n")
|
||||
return 1
|
||||
thread = thread_from_dict(data)
|
||||
result = download_media(thread, args.output, args.cookies)
|
||||
if not result.is_ok:
|
||||
sys.stderr.write(f"error: {result.error.kind}: {result.error.detail}\n")
|
||||
return 1
|
||||
print(f"downloaded {len(result.data)} file(s) to {args.output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Shared dataclasses for the scripts.twitter_threads namespace."""
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Generic, TypeVar
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ErrorInfo:
|
||||
kind: str
|
||||
source: str
|
||||
detail: str
|
||||
|
||||
def make_error(kind: str, source: str, detail: str) -> ErrorInfo:
|
||||
return ErrorInfo(kind=kind, source=source, detail=detail)
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PostMetrics:
|
||||
reply_count: int
|
||||
repost_count: int
|
||||
like_count: int
|
||||
view_count: int | None
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PostData:
|
||||
post_id: str
|
||||
author: str
|
||||
handle: str
|
||||
text: str
|
||||
timestamp: str
|
||||
media_urls: tuple[str, ...]
|
||||
reply_to_id: str | None
|
||||
quote_of_id: str | None
|
||||
metrics: PostMetrics
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ThreadData:
|
||||
root_post_id: str
|
||||
posts: tuple[PostData, ...]
|
||||
source_url: str
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Result(Generic[T]):
|
||||
data: T | None = None
|
||||
error: ErrorInfo | None = None
|
||||
|
||||
@property
|
||||
def is_ok(self) -> bool:
|
||||
return self.error is None
|
||||
|
||||
@classmethod
|
||||
def ok(cls, data: T) -> "Result[T]":
|
||||
return cls(data=data, error=None)
|
||||
|
||||
@classmethod
|
||||
def err(cls, error: ErrorInfo) -> "Result[T]":
|
||||
return cls(data=None, error=error)
|
||||
|
||||
def thread_from_dict(d: dict[str, Any]) -> ThreadData:
|
||||
posts: list[PostData] = []
|
||||
for pd in d.get("posts", []):
|
||||
m = pd.get("metrics", {})
|
||||
metrics = PostMetrics(
|
||||
reply_count=int(m.get("reply_count", 0)),
|
||||
repost_count=int(m.get("repost_count", 0)),
|
||||
like_count=int(m.get("like_count", 0)),
|
||||
view_count=m.get("view_count"),
|
||||
)
|
||||
posts.append(PostData(
|
||||
post_id=pd.get("post_id", ""),
|
||||
author=pd.get("author", ""),
|
||||
handle=pd.get("handle", ""),
|
||||
text=pd.get("text", ""),
|
||||
timestamp=pd.get("timestamp", ""),
|
||||
media_urls=tuple(pd.get("media_urls", [])),
|
||||
reply_to_id=pd.get("reply_to_id"),
|
||||
quote_of_id=pd.get("quote_of_id"),
|
||||
metrics=metrics,
|
||||
))
|
||||
return ThreadData(root_post_id=d.get("root_post_id", ""), posts=tuple(posts), source_url=d.get("source_url", ""))
|
||||
@@ -0,0 +1,264 @@
|
||||
"""Acquire a Twitter/X thread into a ThreadData: gallery-dl (URL) or html.parser (local HTML)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import asdict
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
|
||||
try:
|
||||
from scripts.twitter_threads.error_types import ErrorInfo, make_error, Result, PostMetrics, PostData, ThreadData
|
||||
except ImportError:
|
||||
from error_types import ErrorInfo, make_error, Result, PostMetrics, PostData, ThreadData
|
||||
|
||||
|
||||
def _normalize_url(url: str) -> str:
|
||||
parts = urlparse(url)
|
||||
return urlunparse((parts.scheme, parts.netloc, parts.path, parts.params, "", ""))
|
||||
|
||||
|
||||
def _to_int(value: str | None, default: int | None) -> int | None:
|
||||
if value is None or value == "":
|
||||
return default
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
class _ThreadHTMLParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.posts: list[PostData] = []
|
||||
self.source_url: str = ""
|
||||
self._attrs: dict[str, str] = {}
|
||||
self._media: list[str] = []
|
||||
self._timestamp: str = ""
|
||||
self._text_parts: list[str] = []
|
||||
self._in_article: bool = False
|
||||
self._in_text: bool = False
|
||||
self._text_depth: int = 0
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
a = {k: (v or "") for k, v in attrs}
|
||||
if tag == "link" and a.get("rel") == "canonical" and a.get("href"):
|
||||
self.source_url = a["href"]
|
||||
if tag == "article" and "data-post-id" in a:
|
||||
self._in_article = True
|
||||
self._attrs = a
|
||||
self._media = []
|
||||
self._timestamp = ""
|
||||
self._text_parts = []
|
||||
self._in_text = False
|
||||
self._text_depth = 0
|
||||
if not self._in_article:
|
||||
return
|
||||
if tag == "time" and a.get("datetime"):
|
||||
self._timestamp = a["datetime"]
|
||||
if tag == "div" and a.get("data-testid") == "tweetText":
|
||||
self._in_text = True
|
||||
self._text_depth = 1
|
||||
self._text_parts = []
|
||||
elif tag == "div" and self._in_text:
|
||||
self._text_depth += 1
|
||||
if tag in ("img", "source", "video") and a.get("src"):
|
||||
self._media.append(a["src"])
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._in_text:
|
||||
self._text_parts.append(data)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag == "div" and self._in_text:
|
||||
self._text_depth -= 1
|
||||
if self._text_depth <= 0:
|
||||
self._in_text = False
|
||||
if tag == "article" and self._in_article:
|
||||
self.posts.append(self._build())
|
||||
self._in_article = False
|
||||
|
||||
def _build(self) -> PostData:
|
||||
a = self._attrs
|
||||
metrics = PostMetrics(
|
||||
reply_count=_to_int(a.get("data-reply-count"), 0),
|
||||
repost_count=_to_int(a.get("data-repost-count"), 0),
|
||||
like_count=_to_int(a.get("data-like-count"), 0),
|
||||
view_count=_to_int(a.get("data-view-count"), None),
|
||||
)
|
||||
return PostData(
|
||||
post_id=a.get("data-post-id", ""),
|
||||
author=a.get("data-author", ""),
|
||||
handle=a.get("data-handle", ""),
|
||||
text="".join(self._text_parts).strip(),
|
||||
timestamp=self._timestamp,
|
||||
media_urls=tuple(self._media),
|
||||
reply_to_id=a.get("data-reply-to"),
|
||||
quote_of_id=a.get("data-quote-of"),
|
||||
metrics=metrics,
|
||||
)
|
||||
|
||||
|
||||
def fetch_thread_from_html(html_path: Path) -> Result[ThreadData]:
|
||||
try:
|
||||
text = Path(html_path).read_text(encoding="utf-8")
|
||||
except OSError as e:
|
||||
return Result.err(make_error("ReadError", "fetch_thread_from_html", str(e)))
|
||||
parser = _ThreadHTMLParser()
|
||||
try:
|
||||
parser.feed(text)
|
||||
parser.close()
|
||||
except Exception as e:
|
||||
return Result.err(make_error("ParseError", "fetch_thread_from_html", str(e)))
|
||||
if not parser.posts:
|
||||
return Result.err(make_error("ParseError", "fetch_thread_from_html", "no <article> elements found"))
|
||||
posts = tuple(parser.posts)
|
||||
return Result.ok(ThreadData(root_post_id=posts[0].post_id, posts=posts, source_url=parser.source_url))
|
||||
|
||||
|
||||
def _find_tweet_meta(entry: Any) -> dict[str, Any] | None:
|
||||
if isinstance(entry, dict):
|
||||
if "tweet_id" in entry or ("content" in entry and "author" in entry):
|
||||
return entry
|
||||
return None
|
||||
if isinstance(entry, (list, tuple)):
|
||||
for item in entry:
|
||||
meta = _find_tweet_meta(item)
|
||||
if meta is not None:
|
||||
return meta
|
||||
return None
|
||||
|
||||
|
||||
def _media_url_from_entry(entry: Any) -> str:
|
||||
if isinstance(entry, (list, tuple)):
|
||||
for item in entry:
|
||||
if isinstance(item, str) and ("pbs.twimg.com" in item or "video.twimg.com" in item):
|
||||
return item
|
||||
return ""
|
||||
|
||||
|
||||
def _post_from_meta(meta: dict[str, Any]) -> PostData:
|
||||
author = meta.get("author") or {}
|
||||
if isinstance(author, dict):
|
||||
author_name = str(author.get("nick") or author.get("name") or "")
|
||||
handle = str(author.get("name") or author.get("nick") or "")
|
||||
else:
|
||||
author_name = str(author)
|
||||
handle = str(author)
|
||||
view = meta.get("view_count")
|
||||
metrics = PostMetrics(
|
||||
reply_count=int(meta.get("reply_count") or 0),
|
||||
repost_count=int(meta.get("retweet_count") or 0),
|
||||
like_count=int(meta.get("favorite_count") or 0),
|
||||
view_count=int(view) if view is not None else None,
|
||||
)
|
||||
return PostData(
|
||||
post_id=str(meta.get("tweet_id") or meta.get("id") or ""),
|
||||
author=author_name,
|
||||
handle=handle,
|
||||
text=str(meta.get("content") or ""),
|
||||
timestamp=str(meta.get("date") or ""),
|
||||
media_urls=(),
|
||||
reply_to_id=str(meta["reply_id"]) if meta.get("reply_id") else None,
|
||||
quote_of_id=str(meta["quote_id"]) if meta.get("quote_id") else None,
|
||||
metrics=metrics,
|
||||
)
|
||||
|
||||
|
||||
def _posts_from_gallery_dl(data: Any) -> list[PostData]:
|
||||
by_id: dict[str, PostData] = {}
|
||||
media_by_id: dict[str, list[str]] = {}
|
||||
order: list[str] = []
|
||||
entries = data if isinstance(data, list) else []
|
||||
for entry in entries:
|
||||
meta = _find_tweet_meta(entry)
|
||||
if meta is None:
|
||||
continue
|
||||
tid = str(meta.get("tweet_id") or meta.get("id") or "")
|
||||
if not tid:
|
||||
continue
|
||||
if tid not in by_id:
|
||||
by_id[tid] = _post_from_meta(meta)
|
||||
media_by_id[tid] = []
|
||||
order.append(tid)
|
||||
media_url = _media_url_from_entry(entry)
|
||||
if media_url:
|
||||
media_by_id[tid].append(media_url)
|
||||
posts: list[PostData] = []
|
||||
for tid in order:
|
||||
base = by_id[tid]
|
||||
posts.append(PostData(
|
||||
post_id=base.post_id,
|
||||
author=base.author,
|
||||
handle=base.handle,
|
||||
text=base.text,
|
||||
timestamp=base.timestamp,
|
||||
media_urls=tuple(media_by_id[tid]),
|
||||
reply_to_id=base.reply_to_id,
|
||||
quote_of_id=base.quote_of_id,
|
||||
metrics=base.metrics,
|
||||
))
|
||||
return posts
|
||||
|
||||
|
||||
def fetch_thread_from_url(url: str, cookies_path: Path | None = None) -> Result[ThreadData]:
|
||||
clean = _normalize_url(url)
|
||||
args = ["gallery-dl", "--dump-json", "-o", "text-tweets=true", "-o", "conversations=true"]
|
||||
if cookies_path is not None:
|
||||
args += ["--cookies", str(cookies_path)]
|
||||
args.append(clean)
|
||||
try:
|
||||
completed = subprocess.run(args, capture_output=True, text=True)
|
||||
except OSError as e:
|
||||
return Result.err(make_error("GalleryDlError", "fetch_thread_from_url", str(e)))
|
||||
if completed.returncode != 0:
|
||||
return Result.err(make_error("GalleryDlError", "fetch_thread_from_url", completed.stderr[:500]))
|
||||
try:
|
||||
data = json.loads(completed.stdout)
|
||||
except json.JSONDecodeError as e:
|
||||
return Result.err(make_error("JsonParseError", "fetch_thread_from_url", str(e)))
|
||||
posts = _posts_from_gallery_dl(data)
|
||||
if not posts:
|
||||
return Result.err(make_error("ParseError", "fetch_thread_from_url", "no tweet metadata in gallery-dl output"))
|
||||
posts.sort(key=lambda p: int(p.post_id) if p.post_id.isdigit() else 0)
|
||||
target_id = clean.rstrip("/").split("/")[-1]
|
||||
root_id = target_id if target_id.isdigit() else posts[0].post_id
|
||||
return Result.ok(ThreadData(root_post_id=root_id, posts=tuple(posts), source_url=clean))
|
||||
|
||||
|
||||
def thread_to_dict(thread: ThreadData) -> dict[str, Any]:
|
||||
return asdict(thread)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Fetch a Twitter/X thread into a ThreadData JSON.")
|
||||
parser.add_argument("source", help="X.com URL or local HTML file path")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output directory")
|
||||
parser.add_argument("--cookies", type=Path, default=None, help="cookies.txt for gallery-dl auth")
|
||||
args = parser.parse_args(argv)
|
||||
if args.source.startswith("http://") or args.source.startswith("https://"):
|
||||
result = fetch_thread_from_url(args.source, args.cookies)
|
||||
else:
|
||||
result = fetch_thread_from_html(Path(args.source))
|
||||
if not result.is_ok:
|
||||
sys.stderr.write(f"error: {result.error.kind}: {result.error.detail}" + "`r`n")
|
||||
return 1
|
||||
thread = result.data
|
||||
out_dir = args.output / thread.root_post_id
|
||||
try:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_path = out_dir / "thread_data.json"
|
||||
out_path.write_text(json.dumps(thread_to_dict(thread), indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
except OSError as e:
|
||||
sys.stderr.write(f"error: WriteError: {e}" + "`r`n")
|
||||
return 1
|
||||
print(str(out_path))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Render ThreadData into a Markdown file with YAML front-matter."""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
try:
|
||||
from scripts.twitter_threads.error_types import ErrorInfo, make_error, Result, ThreadData, PostData, thread_from_dict
|
||||
except ImportError:
|
||||
from error_types import ErrorInfo, make_error, Result, ThreadData, PostData, thread_from_dict
|
||||
|
||||
_EM_DASH = "\u2014"
|
||||
|
||||
def _build_id_to_index(thread: ThreadData) -> dict[str, int]:
|
||||
return {post.post_id: i + 1 for i, post in enumerate(thread.posts)}
|
||||
|
||||
def _format_view_count(value: int | None) -> str:
|
||||
if value is None:
|
||||
return "null"
|
||||
return str(value)
|
||||
|
||||
def _root_post(thread: ThreadData) -> PostData:
|
||||
for p in thread.posts:
|
||||
if p.post_id == thread.root_post_id:
|
||||
return p
|
||||
return thread.posts[0]
|
||||
|
||||
def _render_yaml(thread: ThreadData, title: str) -> list[str]:
|
||||
root = _root_post(thread)
|
||||
return [
|
||||
"---",
|
||||
f'title: "{title}"',
|
||||
f'author: "{root.author}"',
|
||||
f'handle: "@{root.handle}"',
|
||||
f'post_url: "{thread.source_url}"',
|
||||
f'post_id: "{thread.root_post_id}"',
|
||||
f'timestamp: "{root.timestamp}"',
|
||||
f'post_count: {len(thread.posts)}',
|
||||
f'reply_count: {root.metrics.reply_count}',
|
||||
f'repost_count: {root.metrics.repost_count}',
|
||||
f'like_count: {root.metrics.like_count}',
|
||||
f'view_count: {_format_view_count(root.metrics.view_count)}',
|
||||
"---",
|
||||
]
|
||||
|
||||
def _media_line(index: int, name: str) -> str:
|
||||
ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
|
||||
if ext in ("mp4", "mov", "webm", "m4v"):
|
||||
return f'<video controls src="./media/{name}"></video>'
|
||||
if ext in ("jpg", "jpeg", "png", "gif", "webp", "bmp"):
|
||||
return f""
|
||||
return f"[Media {index}](./media/{name})"
|
||||
|
||||
def _render_post(post: PostData, n: int, id_to_index: dict[str, int], media_names: dict[str, list[str]]) -> list[str]:
|
||||
lines: list[str] = []
|
||||
header = f"## Post {n} ({post.timestamp})"
|
||||
if post.reply_to_id is not None and post.reply_to_id in id_to_index:
|
||||
header += f" {_EM_DASH} reply to Post {id_to_index[post.reply_to_id]}"
|
||||
lines.append(header)
|
||||
lines.append("")
|
||||
lines.append(post.text)
|
||||
lines.append("")
|
||||
if post.quote_of_id is not None:
|
||||
lines.append(f"> [Quoting post {post.quote_of_id}](https://x.com/i/web/status/{post.quote_of_id})")
|
||||
lines.append("")
|
||||
names = media_names.get(post.post_id, [])
|
||||
if names:
|
||||
for k, name in enumerate(names, start=1):
|
||||
lines.append(_media_line(k, name))
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
def render_markdown(thread: ThreadData, media_names: dict[str, list[str]], output: Path) -> Result[Path]:
|
||||
lines: list[str] = []
|
||||
id_to_index = _build_id_to_index(thread)
|
||||
root = _root_post(thread)
|
||||
first_line = root.text.splitlines()[0] if root.text.splitlines() else ""
|
||||
title = first_line[:80]
|
||||
lines.extend(_render_yaml(thread, title))
|
||||
lines.append("")
|
||||
lines.append(f"# @{root.handle} {_EM_DASH} {title}")
|
||||
lines.append("")
|
||||
for i, post in enumerate(thread.posts):
|
||||
n = i + 1
|
||||
lines.extend(_render_post(post, n, id_to_index, media_names))
|
||||
content = "\n".join(lines)
|
||||
try:
|
||||
output.write_text(content, encoding="utf-8")
|
||||
except OSError as e:
|
||||
return Result.err(make_error("WriteError", "render_markdown", str(e)))
|
||||
return Result.ok(output)
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Render a Twitter/X thread_data.json to Markdown.")
|
||||
parser.add_argument("--input", type=Path, required=True, help="thread_data.json path")
|
||||
parser.add_argument("--media-dir", type=Path, required=True, help="media directory to link")
|
||||
parser.add_argument("--output", type=Path, required=True, help="output .md path")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
raw = args.input.read_text(encoding="utf-8")
|
||||
except OSError as e:
|
||||
sys.stderr.write(f"error: ReadError: {e}\n")
|
||||
return 1
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
sys.stderr.write(f"error: JsonParseError: {e}\n")
|
||||
return 1
|
||||
thread = thread_from_dict(data)
|
||||
media_names: dict[str, list[str]] = {}
|
||||
for post in thread.posts:
|
||||
if args.media_dir.exists():
|
||||
files = sorted(f.name for f in args.media_dir.glob(f"{post.post_id}_*"))
|
||||
if files:
|
||||
media_names[post.post_id] = files
|
||||
result = render_markdown(thread, media_names, args.output)
|
||||
if not result.is_ok:
|
||||
sys.stderr.write(f"error: {result.error.kind}: {result.error.detail}\n")
|
||||
return 1
|
||||
print(str(args.output))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,133 @@
|
||||
"""RED tests for scripts.twitter_threads.fetch_thread (normalize URL + HTML to ThreadData)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.twitter_threads.error_types import PostData, ThreadData
|
||||
from scripts.twitter_threads.fetch_thread import fetch_thread_from_html, _normalize_url
|
||||
|
||||
|
||||
def test_normalize_url_strips_query() -> None:
|
||||
assert _normalize_url("https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20") == "https://x.com/VPCOMPRESSB/status/1991383117571957052"
|
||||
assert _normalize_url("https://x.com/NOTimothyLottes/status/123") == "https://x.com/NOTimothyLottes/status/123"
|
||||
|
||||
|
||||
def test_parse_html_single_post(tmp_path: Path) -> None:
|
||||
html = """<html><body>
|
||||
<link rel="canonical" href="https://x.com/NOTimothyLottes/status/1757198624818168210">
|
||||
<article data-post-id="1757198624818168210" data-author="Timothy Lottes" data-handle="NOTimothyLottes" data-reply-count="5" data-repost-count="10" data-like-count="100" data-view-count="5000">
|
||||
<time datetime="2024-02-13T12:00:00Z">Feb 13</time>
|
||||
<div data-testid="tweetText">Hello thread</div>
|
||||
<img src="https://pbs.twimg.com/media/AAA.jpg">
|
||||
</article>
|
||||
</body></html>"""
|
||||
p = tmp_path / "x.html"
|
||||
p.write_text(html, encoding="utf-8")
|
||||
result = fetch_thread_from_html(p)
|
||||
assert result.is_ok
|
||||
td = result.data
|
||||
assert td is not None
|
||||
assert td.root_post_id == "1757198624818168210"
|
||||
assert len(td.posts) == 1
|
||||
post = td.posts[0]
|
||||
assert post.post_id == "1757198624818168210"
|
||||
assert post.author == "Timothy Lottes"
|
||||
assert post.handle == "NOTimothyLottes"
|
||||
assert post.timestamp == "2024-02-13T12:00:00Z"
|
||||
assert "Hello thread" in post.text
|
||||
assert post.media_urls == ("https://pbs.twimg.com/media/AAA.jpg",)
|
||||
assert post.metrics.reply_count == 5
|
||||
assert post.metrics.repost_count == 10
|
||||
assert post.metrics.like_count == 100
|
||||
assert post.metrics.view_count == 5000
|
||||
assert td.source_url == "https://x.com/NOTimothyLottes/status/1757198624818168210"
|
||||
|
||||
|
||||
def test_parse_html_thread(tmp_path: Path) -> None:
|
||||
html = """<html><body>
|
||||
<article data-post-id="1" data-author="A" data-handle="a">
|
||||
<time datetime="2024-01-01T00:00:00Z">d1</time>
|
||||
<div data-testid="tweetText">p1</div>
|
||||
</article>
|
||||
<article data-post-id="2" data-author="A" data-handle="a" data-reply-to="1">
|
||||
<time datetime="2024-01-02T00:00:00Z">d2</time>
|
||||
<div data-testid="tweetText">p2</div>
|
||||
</article>
|
||||
<article data-post-id="3" data-author="A" data-handle="a" data-reply-to="2">
|
||||
<time datetime="2024-01-03T00:00:00Z">d3</time>
|
||||
<div data-testid="tweetText">p3</div>
|
||||
</article>
|
||||
</body></html>"""
|
||||
p = tmp_path / "x.html"
|
||||
p.write_text(html, encoding="utf-8")
|
||||
result = fetch_thread_from_html(p)
|
||||
assert result.is_ok
|
||||
td = result.data
|
||||
assert td is not None
|
||||
assert len(td.posts) == 3
|
||||
ids = [pt.post_id for pt in td.posts]
|
||||
assert ids == ["1", "2", "3"]
|
||||
assert td.posts[1].reply_to_id == "1"
|
||||
assert td.posts[2].reply_to_id == "2"
|
||||
assert td.posts[0].reply_to_id is None
|
||||
|
||||
|
||||
def test_parse_html_no_media(tmp_path: Path) -> None:
|
||||
html = """<html><body>
|
||||
<article data-post-id="1" data-author="A" data-handle="a">
|
||||
<time datetime="2024-01-01T00:00:00Z">d</time>
|
||||
<div data-testid="tweetText">no media here</div>
|
||||
</article>
|
||||
</body></html>"""
|
||||
p = tmp_path / "x.html"
|
||||
p.write_text(html, encoding="utf-8")
|
||||
result = fetch_thread_from_html(p)
|
||||
assert result.is_ok
|
||||
td = result.data
|
||||
assert td is not None
|
||||
assert td.posts[0].media_urls == ()
|
||||
|
||||
|
||||
def test_parse_html_quote(tmp_path: Path) -> None:
|
||||
html = """<html><body>
|
||||
<article data-post-id="1" data-author="A" data-handle="a" data-quote-of="999">
|
||||
<time datetime="2024-01-01T00:00:00Z">d</time>
|
||||
<div data-testid="tweetText">quoting</div>
|
||||
</article>
|
||||
</body></html>"""
|
||||
p = tmp_path / "x.html"
|
||||
p.write_text(html, encoding="utf-8")
|
||||
result = fetch_thread_from_html(p)
|
||||
assert result.is_ok
|
||||
td = result.data
|
||||
assert td is not None
|
||||
assert td.posts[0].quote_of_id == "999"
|
||||
|
||||
|
||||
def test_parse_html_malformed(tmp_path: Path) -> None:
|
||||
html = "<html><body><p>no articles here</p></body></html>"
|
||||
p = tmp_path / "x.html"
|
||||
p.write_text(html, encoding="utf-8")
|
||||
result = fetch_thread_from_html(p)
|
||||
assert result.is_ok is False
|
||||
assert result.error is not None
|
||||
assert result.error.kind == "ParseError"
|
||||
|
||||
|
||||
def test_parse_html_view_count_absent(tmp_path: Path) -> None:
|
||||
html = """<html><body>
|
||||
<article data-post-id="1" data-author="A" data-handle="a">
|
||||
<time datetime="2024-01-01T00:00:00Z">d</time>
|
||||
<div data-testid="tweetText">x</div>
|
||||
</article>
|
||||
</body></html>"""
|
||||
p = tmp_path / "x.html"
|
||||
p.write_text(html, encoding="utf-8")
|
||||
result = fetch_thread_from_html(p)
|
||||
assert result.is_ok
|
||||
td = result.data
|
||||
assert td is not None
|
||||
assert td.posts[0].metrics.view_count is None
|
||||
assert td.posts[0].metrics.reply_count == 0
|
||||
@@ -0,0 +1,62 @@
|
||||
|
||||
"""Tests for scripts.twitter_threads.download_media gallery-dl subprocess contract."""
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
import pytest
|
||||
from scripts.twitter_threads.error_types import PostMetrics, PostData, ThreadData
|
||||
from scripts.twitter_threads.download_media import download_media, media_files_for_post
|
||||
|
||||
|
||||
def _post(post_id: str) -> PostData:
|
||||
metrics = PostMetrics(reply_count=0, repost_count=0, like_count=0, view_count=None)
|
||||
return PostData(post_id=post_id, author="Tim", handle="NOTimothyLottes", text="t", timestamp="2024-02-01T00:00:00Z", media_urls=(), reply_to_id=None, quote_of_id=None, metrics=metrics)
|
||||
|
||||
|
||||
def _thread(post_id: str) -> ThreadData:
|
||||
return ThreadData(root_post_id=post_id, posts=(_post(post_id),), source_url=f"https://x.com/x/status/{post_id}")
|
||||
|
||||
|
||||
def test_download_collects_matching_files(tmp_path: Path) -> None:
|
||||
mediadir = tmp_path / "m"
|
||||
mediadir.mkdir()
|
||||
(mediadir / "123_1.jpg").write_bytes(b"x")
|
||||
(mediadir / "123_2.png").write_bytes(b"x")
|
||||
(mediadir / "999_1.jpg").write_bytes(b"x")
|
||||
with patch("scripts.twitter_threads.download_media.subprocess.run", MagicMock(return_value=MagicMock(returncode=0))):
|
||||
r = download_media(_thread("123"), mediadir)
|
||||
assert r.is_ok
|
||||
names = [p.name for p in r.data]
|
||||
assert "123_1.jpg" in names
|
||||
assert "123_2.png" in names
|
||||
assert "999_1.jpg" not in names
|
||||
|
||||
|
||||
def test_download_invokes_gallery_dl_with_url_and_cookies(tmp_path: Path) -> None:
|
||||
mediadir = tmp_path / "m"
|
||||
ck = tmp_path / "cookies.txt"
|
||||
ck.write_text("x", encoding="utf-8")
|
||||
with patch("scripts.twitter_threads.download_media.subprocess.run", MagicMock(return_value=MagicMock(returncode=0))) as mock:
|
||||
download_media(_thread("55"), mediadir, ck)
|
||||
assert mock.called
|
||||
args = mock.call_args.args[0]
|
||||
assert "gallery-dl" in args[0] or args[0] == "gallery-dl"
|
||||
assert "https://x.com/x/status/55" in args
|
||||
assert "--cookies" in args
|
||||
assert str(ck) in args
|
||||
|
||||
|
||||
def test_download_gallery_dl_error(tmp_path: Path) -> None:
|
||||
with patch("scripts.twitter_threads.download_media.subprocess.run", MagicMock(return_value=MagicMock(returncode=1, stderr="boom"))):
|
||||
r = download_media(_thread("123"), tmp_path / "m")
|
||||
assert r.is_ok is False
|
||||
assert r.error.kind == "GalleryDlError"
|
||||
|
||||
|
||||
def test_media_files_for_post(tmp_path: Path) -> None:
|
||||
d = tmp_path / "m"
|
||||
d.mkdir()
|
||||
(d / "123_1.jpg").write_bytes(b"x")
|
||||
(d / "123_2.png").write_bytes(b"x")
|
||||
(d / "999_1.jpg").write_bytes(b"x")
|
||||
assert media_files_for_post(d, "123") == ["123_1.jpg", "123_2.png"]
|
||||
@@ -0,0 +1,101 @@
|
||||
"""RED tests for the twitter_threads pipeline: thread_from_dict, download_main, render_main."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.twitter_threads.error_types import PostMetrics, PostData, ThreadData, thread_from_dict
|
||||
from scripts.twitter_threads.fetch_thread import fetch_thread_from_html, thread_to_dict
|
||||
from scripts.twitter_threads.download_media import download_media, main as download_main
|
||||
from scripts.twitter_threads.render_markdown import render_markdown, main as render_main
|
||||
|
||||
|
||||
class _FakeResp:
|
||||
def __init__(self, data: bytes) -> None:
|
||||
self._data = data
|
||||
|
||||
def read(self) -> bytes:
|
||||
return self._data
|
||||
|
||||
def __enter__(self) -> "_FakeResp":
|
||||
return self
|
||||
|
||||
def __exit__(self, *a: object) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _post(post_id: str, text: str = "hi", media_urls: tuple[str, ...] = (), reply_to_id: str | None = None) -> PostData:
|
||||
metrics = PostMetrics(reply_count=1, repost_count=2, like_count=3, view_count=None)
|
||||
return PostData(post_id=post_id, author="Tim", handle="NOTimothyLottes", text=text, timestamp="2024-02-01T00:00:00Z", media_urls=media_urls, reply_to_id=reply_to_id, quote_of_id=None, metrics=metrics)
|
||||
|
||||
|
||||
def test_thread_json_roundtrip() -> None:
|
||||
p1 = _post("1")
|
||||
p2 = _post("2", media_urls=("https://pbs.twimg.com/media/AAA.jpg",), reply_to_id="1")
|
||||
t = ThreadData(root_post_id="1", posts=(p1, p2), source_url="https://x.com/x/x/1")
|
||||
d = thread_to_dict(t)
|
||||
t2 = thread_from_dict(json.loads(json.dumps(d)))
|
||||
assert t2 == t
|
||||
|
||||
|
||||
def test_pipeline_html_to_markdown(tmp_path: Path) -> None:
|
||||
html = """<html><body>
|
||||
<article data-post-id="10" data-author="Tim" data-handle="NOTimothyLottes">
|
||||
<time datetime="2024-02-01T00:00:00Z">d1</time>
|
||||
<div data-testid="tweetText">first</div>
|
||||
<img src="https://pbs.twimg.com/media/ZZZ.jpg">
|
||||
</article>
|
||||
<article data-post-id="11" data-author="Tim" data-handle="NOTimothyLottes" data-reply-to="10">
|
||||
<time datetime="2024-02-01T00:00:00Z">d2</time>
|
||||
<div data-testid="tweetText">second</div>
|
||||
</article>
|
||||
</body></html>"""
|
||||
path = tmp_path / "thread.html"
|
||||
path.write_text(html, encoding="utf-8")
|
||||
res = fetch_thread_from_html(path)
|
||||
assert res.is_ok
|
||||
td = res.data
|
||||
assert td is not None
|
||||
media_names = {"10": ["10_img1.jpg"]}
|
||||
out = tmp_path / "thread.md"
|
||||
rr = render_markdown(td, media_names, out)
|
||||
assert rr.is_ok
|
||||
text = out.read_text(encoding="utf-8")
|
||||
assert "## Post 1" in text
|
||||
assert "## Post 2" in text
|
||||
assert "reply to Post 1" in text
|
||||
assert "" in text
|
||||
|
||||
|
||||
def test_download_cli(tmp_path: Path) -> None:
|
||||
p1 = _post("5", media_urls=("https://pbs.twimg.com/media/BBB.jpg",))
|
||||
t = ThreadData(root_post_id="5", posts=(p1,), source_url="https://x.com/x/x/5")
|
||||
thread_data_json = tmp_path / "thread_data.json"
|
||||
thread_data_json.write_text(json.dumps(thread_to_dict(t)), encoding="utf-8")
|
||||
mediadir = tmp_path / "media"
|
||||
mediadir.mkdir()
|
||||
(mediadir / "5_1.jpg").write_bytes(b"x")
|
||||
with patch("scripts.twitter_threads.download_media.subprocess.run", MagicMock(return_value=MagicMock(returncode=0))):
|
||||
code = download_main(["--input", str(thread_data_json), "--output", str(mediadir)])
|
||||
assert code == 0
|
||||
assert (mediadir / "5_1.jpg").exists()
|
||||
|
||||
|
||||
def test_render_cli(tmp_path: Path) -> None:
|
||||
p1 = _post("7", text="body", media_urls=("https://pbs.twimg.com/media/CCC.jpg",))
|
||||
t = ThreadData(root_post_id="7", posts=(p1,), source_url="https://x.com/x/x/7")
|
||||
thread_data_json = tmp_path / "thread_data.json"
|
||||
thread_data_json.write_text(json.dumps(thread_to_dict(t)), encoding="utf-8")
|
||||
mediadir = tmp_path / "media"
|
||||
mediadir.mkdir()
|
||||
(mediadir / "7_img1.jpg").write_bytes(b"x")
|
||||
out = tmp_path / "thread.md"
|
||||
code = render_main(["--input", str(thread_data_json), "--media-dir", str(mediadir), "--output", str(out)])
|
||||
assert code == 0
|
||||
assert out.exists()
|
||||
text = out.read_text(encoding="utf-8")
|
||||
assert "[Media 1](./media/7_img1.jpg)" in text
|
||||
assert "body" in text
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Tests for scripts.twitter_threads.render_markdown and Result contract."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.twitter_threads.error_types import ErrorInfo, make_error, PostMetrics, PostData, Result, ThreadData
|
||||
from scripts.twitter_threads.render_markdown import render_markdown
|
||||
|
||||
|
||||
def _post(post_id: str, text: str, reply_to_id: str | None = None, quote_of_id: str | None = None, view_count: int = 0, timestamp: str = "2024-02-01T00:00:00Z", handle: str = "NOTimothyLottes", author: str = "Tim") -> PostData:
|
||||
metrics = PostMetrics(reply_count=0, repost_count=0, like_count=0, view_count=view_count)
|
||||
return PostData(post_id=post_id, author=author, handle=handle, text=text, timestamp=timestamp, media_urls=(), reply_to_id=reply_to_id, quote_of_id=quote_of_id, metrics=metrics)
|
||||
|
||||
|
||||
def test_result_ok_and_err() -> None:
|
||||
r = Result.ok("x")
|
||||
assert r.is_ok is True
|
||||
assert r.data == "x"
|
||||
e = make_error("K", "s", "d")
|
||||
r2 = Result.err(e)
|
||||
assert r2.is_ok is False
|
||||
assert r2.error is e
|
||||
|
||||
|
||||
def test_render_single_post(tmp_path: Path) -> None:
|
||||
root = _post("123", "Hello world", view_count=5)
|
||||
thread = ThreadData(root_post_id="123", posts=(root,), source_url="https://x.com/x/x/123")
|
||||
out = tmp_path / "thread.md"
|
||||
result = render_markdown(thread, {}, out)
|
||||
assert result.is_ok
|
||||
assert result.data == out
|
||||
assert out.exists()
|
||||
text = out.read_text(encoding="utf-8")
|
||||
assert text.startswith("---")
|
||||
assert "@NOTimothyLottes" in text
|
||||
assert "post_id:" in text
|
||||
assert "post_count: 1" in text
|
||||
assert "view_count: 5" in text
|
||||
assert "Hello world" in text
|
||||
|
||||
|
||||
def test_render_thread(tmp_path: Path) -> None:
|
||||
p1 = _post("1", "first post")
|
||||
p2 = _post("2", "second post", reply_to_id="1")
|
||||
p3 = _post("3", "third post", reply_to_id="2")
|
||||
thread = ThreadData(root_post_id="1", posts=(p1, p2, p3), source_url="https://x.com/x/x/1")
|
||||
out = tmp_path / "thread.md"
|
||||
result = render_markdown(thread, {}, out)
|
||||
assert result.is_ok
|
||||
text = out.read_text(encoding="utf-8")
|
||||
i1 = text.index("## Post 1")
|
||||
i2 = text.index("## Post 2")
|
||||
i3 = text.index("## Post 3")
|
||||
assert i1 < i2 < i3
|
||||
assert "reply to Post 1" in text
|
||||
assert "reply to Post 2" in text
|
||||
|
||||
|
||||
def test_render_quote_tweet(tmp_path: Path) -> None:
|
||||
root = _post("100", "this is a quote", quote_of_id="999")
|
||||
thread = ThreadData(root_post_id="100", posts=(root,), source_url="https://x.com/x/x/100")
|
||||
out = tmp_path / "thread.md"
|
||||
result = render_markdown(thread, {}, out)
|
||||
assert result.is_ok
|
||||
text = out.read_text(encoding="utf-8")
|
||||
quote_lines = [line for line in text.splitlines() if line.startswith("> ")]
|
||||
assert any("Quoting" in line and "999" in line for line in quote_lines)
|
||||
|
||||
|
||||
def test_render_media_links(tmp_path: Path) -> None:
|
||||
root = _post("123", "media post")
|
||||
thread = ThreadData(root_post_id="123", posts=(root,), source_url="https://x.com/x/x/123")
|
||||
out = tmp_path / "thread.md"
|
||||
media_names = {"123": ["123_img1.jpg", "123_img2.jpg", "123_vid1.mp4"]}
|
||||
result = render_markdown(thread, media_names, out)
|
||||
assert result.is_ok
|
||||
text = out.read_text(encoding="utf-8")
|
||||
assert "" in text
|
||||
assert "" in text
|
||||
assert '<video controls src="./media/123_vid1.mp4"></video>' in text
|
||||
|
||||
|
||||
def test_render_view_count_null(tmp_path: Path) -> None:
|
||||
root = _post("123", "hello", view_count=None)
|
||||
thread = ThreadData(root_post_id="123", posts=(root,), source_url="https://x.com/x/x/123")
|
||||
out = tmp_path / "thread.md"
|
||||
result = render_markdown(thread, {}, out)
|
||||
assert result.is_ok
|
||||
text = out.read_text(encoding="utf-8")
|
||||
assert "view_count: null" in text
|
||||
|
||||
|
||||
def test_render_title_truncated(tmp_path: Path) -> None:
|
||||
text100 = "A" * 100
|
||||
root = _post("123", text100)
|
||||
thread = ThreadData(root_post_id="123", posts=(root,), source_url="https://x.com/x/x/123")
|
||||
out = tmp_path / "thread.md"
|
||||
result = render_markdown(thread, {}, out)
|
||||
assert result.is_ok
|
||||
text = out.read_text(encoding="utf-8")
|
||||
h1_lines = [line for line in text.splitlines() if line.startswith("# @")]
|
||||
assert h1_lines, "expected an H1 line starting with '# @'"
|
||||
h1 = h1_lines[0]
|
||||
assert "A" * 80 in h1
|
||||
assert "A" * 81 not in h1
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Tests for scripts.twitter_threads.error_types dataclass contract."""
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.twitter_threads.error_types import ErrorInfo, make_error, PostMetrics, PostData, ThreadData
|
||||
|
||||
|
||||
def test_error_info_fields() -> None:
|
||||
e = ErrorInfo(kind="ParseError", source="fetch", detail="bad html")
|
||||
assert e.kind == "ParseError"
|
||||
assert e.source == "fetch"
|
||||
assert e.detail == "bad html"
|
||||
|
||||
|
||||
def test_make_error_factory() -> None:
|
||||
e = make_error("HttpError", "download_media", "404")
|
||||
assert isinstance(e, ErrorInfo)
|
||||
assert e.kind == "HttpError"
|
||||
assert e.source == "download_media"
|
||||
assert e.detail == "404"
|
||||
|
||||
|
||||
def test_error_info_frozen() -> None:
|
||||
e = ErrorInfo(kind="x", source="y", detail="z")
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
e.kind = "w"
|
||||
|
||||
|
||||
def test_error_info_slots() -> None:
|
||||
e = ErrorInfo(kind="x", source="y", detail="z")
|
||||
assert not hasattr(e, "__dict__")
|
||||
|
||||
|
||||
def test_post_metrics() -> None:
|
||||
m = PostMetrics(reply_count=1, repost_count=2, like_count=3, view_count=None)
|
||||
assert m.reply_count == 1
|
||||
assert m.repost_count == 2
|
||||
assert m.like_count == 3
|
||||
assert m.view_count is None
|
||||
m2 = PostMetrics(reply_count=10, repost_count=20, like_count=30, view_count=100)
|
||||
assert m2.view_count == 100
|
||||
|
||||
|
||||
def test_post_data_fields() -> None:
|
||||
metrics = PostMetrics(reply_count=1, repost_count=2, like_count=42, view_count=None)
|
||||
p = PostData(
|
||||
post_id="123",
|
||||
author="Tim",
|
||||
handle="NOTimothyLottes",
|
||||
text="hi",
|
||||
timestamp="2024-02-01T00:00:00Z",
|
||||
media_urls=("https://x/i.jpg",),
|
||||
reply_to_id=None,
|
||||
quote_of_id=None,
|
||||
metrics=metrics,
|
||||
)
|
||||
assert p.post_id == "123"
|
||||
assert p.author == "Tim"
|
||||
assert p.handle == "NOTimothyLottes"
|
||||
assert p.text == "hi"
|
||||
assert p.timestamp == "2024-02-01T00:00:00Z"
|
||||
assert isinstance(p.media_urls, tuple)
|
||||
assert p.media_urls == ("https://x/i.jpg",)
|
||||
assert p.reply_to_id is None
|
||||
assert p.quote_of_id is None
|
||||
assert p.metrics.like_count == 42
|
||||
|
||||
|
||||
def test_post_data_slots_frozen() -> None:
|
||||
metrics = PostMetrics(reply_count=1, repost_count=2, like_count=3, view_count=None)
|
||||
p = PostData(
|
||||
post_id="123",
|
||||
author="Tim",
|
||||
handle="NOTimothyLottes",
|
||||
text="hi",
|
||||
timestamp="2024-02-01T00:00:00Z",
|
||||
media_urls=(),
|
||||
reply_to_id=None,
|
||||
quote_of_id=None,
|
||||
metrics=metrics,
|
||||
)
|
||||
assert not hasattr(p, "__dict__")
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
p.post_id = "456"
|
||||
|
||||
|
||||
def test_thread_data_fields() -> None:
|
||||
metrics = PostMetrics(reply_count=1, repost_count=2, like_count=3, view_count=None)
|
||||
p = PostData(
|
||||
post_id="123",
|
||||
author="Tim",
|
||||
handle="NOTimothyLottes",
|
||||
text="hi",
|
||||
timestamp="2024-02-01T00:00:00Z",
|
||||
media_urls=(),
|
||||
reply_to_id=None,
|
||||
quote_of_id=None,
|
||||
metrics=metrics,
|
||||
)
|
||||
t = ThreadData(root_post_id="123", posts=(p,), source_url="https://x.com/NOTimothyLottes/status/123")
|
||||
assert t.root_post_id == "123"
|
||||
assert isinstance(t.posts, tuple)
|
||||
assert len(t.posts) == 1
|
||||
assert t.posts[0] is p
|
||||
assert t.source_url == "https://x.com/NOTimothyLottes/status/123"
|
||||
|
||||
|
||||
def test_thread_data_slots_frozen() -> None:
|
||||
metrics = PostMetrics(reply_count=1, repost_count=2, like_count=3, view_count=None)
|
||||
p = PostData(
|
||||
post_id="123",
|
||||
author="Tim",
|
||||
handle="NOTimothyLottes",
|
||||
text="hi",
|
||||
timestamp="2024-02-01T00:00:00Z",
|
||||
media_urls=(),
|
||||
reply_to_id=None,
|
||||
quote_of_id=None,
|
||||
metrics=metrics,
|
||||
)
|
||||
t = ThreadData(root_post_id="123", posts=(p,), source_url="https://x.com/x/x/123")
|
||||
assert not hasattr(t, "__dict__")
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
t.source_url = "https://other"
|
||||
Reference in New Issue
Block a user