From 53721a77962db3179f9a7f3af723468bee810c9b Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 5 Jul 2026 18:28:40 -0400 Subject: [PATCH] fix(twitter_threads): align gallery-dl parser to live schema + add track extraction scripts Live Task 5.3 validation against real gallery-dl output: (1) pass -o text-tweets=true so text-only tweets are returned (media-only default yielded []); (2) fix author name/nick swap (gallery-dl name=handle, nick=display). Added convert_cookies.py (JSON cookie export -> Netscape) + run_corpus.py (8-URL pipeline -> docs/twitter/) to the track dir. Extracted 8/8 threads. 33 tests still pass. --- .../convert_cookies.py | 41 ++++++++++++ .../run_corpus.py | 66 +++++++++++++++++++ scripts/twitter_threads/fetch_thread.py | 6 +- 3 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py create mode 100644 conductor/tracks/twitter_threads_extraction_20260705/run_corpus.py diff --git a/conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py b/conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py new file mode 100644 index 00000000..17b97109 --- /dev/null +++ b/conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py @@ -0,0 +1,41 @@ +"""Convert a JSON cookie export (Cookie-Editor style) to Netscape cookies.txt for gallery-dl. + +Part of the twitter_threads_extraction_20260705 track. Reads the user's JSON cookie +export at scripts/twitter_threads/cookies.txt (gitignored) and writes a Netscape-format +file to a gitignored location so the secret values never land in git. Prints no cookie values. + +Usage: uv run python conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py +""" +from __future__ import annotations + +import json +from pathlib import Path + +SRC = Path("scripts/twitter_threads/cookies.txt") +DST = Path("tests/artifacts/tier2_state/twitter_threads_extraction_20260705/cookies_netscape.txt") + + +def main() -> int: + data = json.loads(SRC.read_text(encoding="utf-8-sig")) + DST.parent.mkdir(parents=True, exist_ok=True) + out: list[str] = ["# Netscape HTTP Cookie File", "# converted from JSON export for gallery-dl", ""] + for c in data: + domain = c.get("domain", "") + include_sub = "TRUE" if domain.startswith(".") else "FALSE" + path = c.get("path", "/") or "/" + secure = "TRUE" if c.get("secure") else "FALSE" + exp = c.get("expirationDate") or c.get("expiry") or c.get("expires") or 2147483647 + try: + exp = int(float(exp)) + except (TypeError, ValueError): + exp = 2147483647 + name = c.get("name", "") + value = c.get("value", "") + out.append("\t".join([domain, include_sub, path, secure, str(exp), name, value])) + DST.write_text("\n".join(out) + "\n", encoding="utf-8") + print(f"wrote {DST} with {len(data)} cookies (values not shown)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/conductor/tracks/twitter_threads_extraction_20260705/run_corpus.py b/conductor/tracks/twitter_threads_extraction_20260705/run_corpus.py new file mode 100644 index 00000000..87e9d78e --- /dev/null +++ b/conductor/tracks/twitter_threads_extraction_20260705/run_corpus.py @@ -0,0 +1,66 @@ +"""Run the twitter_threads pipeline over the 8-URL corpus (Task 5.3) into ./docs/twitter/. + +Part of the twitter_threads_extraction_20260705 track. Uses the Netscape cookies produced +by convert_cookies.py. Output layout per URL: docs/twitter//{thread.md, thread_data.json, media/}. +Prints a per-URL summary; never prints cookie values. + +Usage: + uv run python conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py + uv run python conductor/tracks/twitter_threads_extraction_20260705/run_corpus.py +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from scripts.twitter_threads.fetch_thread import fetch_thread_from_url, thread_to_dict +from scripts.twitter_threads.download_media import download_media, media_names_for_post +from scripts.twitter_threads.render_markdown import render_markdown + +COOKIES = Path("tests/artifacts/tier2_state/twitter_threads_extraction_20260705/cookies_netscape.txt") +CORPUS = Path("docs/twitter") + +URLS = [ + "https://x.com/NOTimothyLottes/status/1757198624818168210", + "https://x.com/NOTimothyLottes/status/1653570742762479620", + "https://x.com/NOTimothyLottes/status/1917646466417381426", + "https://x.com/NOTimothyLottes/status/1917645859791200562", + "https://x.com/NOTimothyLottes/status/1917644904055910502", + "https://x.com/NOTimothyLottes/status/1917642786804785230", + "https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20", + "https://x.com/VPCOMPRESSB/status/1987744335333622188?s=20", +] + + +def main() -> int: + CORPUS.mkdir(parents=True, exist_ok=True) + ok = 0 + for url in URLS: + fr = fetch_thread_from_url(url, COOKIES) + if not fr.is_ok: + print(f"FAIL fetch {url} -> {fr.error.kind}: {fr.error.detail[:160]}") + continue + td = fr.data + base = CORPUS / td.root_post_id + media_dir = base / "media" + dr = download_media(td, media_dir) + n_media = len(dr.data) if dr.is_ok else -1 + if not dr.is_ok: + print(f"WARN media {td.root_post_id} -> {dr.error.kind}: {dr.error.detail[:120]}") + (base / "thread_data.json").write_text(json.dumps(thread_to_dict(td), indent=2, ensure_ascii=False), encoding="utf-8") + media_names = {p.post_id: media_names_for_post(p) for p in td.posts if p.media_urls} + rr = render_markdown(td, media_names, base / "thread.md") + handle = td.posts[0].handle if td.posts else "?" + status = "OK" if rr.is_ok else f"RENDER_FAIL {rr.error.kind}" + print(f"{status} {td.root_post_id}: posts={len(td.posts)} media={n_media} @{handle} -> {base / 'thread.md'}") + if rr.is_ok: + ok += 1 + print(f"\nCORPUS DONE: {ok}/{len(URLS)} threads rendered into {CORPUS}") + return 0 if ok == len(URLS) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/twitter_threads/fetch_thread.py b/scripts/twitter_threads/fetch_thread.py index fda19e18..b877f355 100644 --- a/scripts/twitter_threads/fetch_thread.py +++ b/scripts/twitter_threads/fetch_thread.py @@ -144,8 +144,8 @@ def _media_url_from_entry(entry: Any) -> str: def _post_from_meta(meta: dict[str, Any]) -> PostData: author = meta.get("author") or {} if isinstance(author, dict): - author_name = str(author.get("name") or author.get("nick") or "") - handle = str(author.get("nick") or author.get("name") or "") + author_name = str(author.get("nick") or author.get("name") or "") + handle = str(author.get("name") or author.get("nick") or "") else: author_name = str(author) handle = str(author) @@ -207,7 +207,7 @@ def _posts_from_gallery_dl(data: Any) -> list[PostData]: def fetch_thread_from_url(url: str, cookies_path: Path | None = None) -> Result[ThreadData]: clean = _normalize_url(url) - args = ["gallery-dl", "--dump-json"] + args = ["gallery-dl", "--dump-json", "-o", "text-tweets=true"] if cookies_path is not None: args += ["--cookies", str(cookies_path)] args.append(clean)