From 7e828cbd61395784ea1f8b097003f520fd91ca77 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 5 Jul 2026 18:50:55 -0400 Subject: [PATCH] refactor(twitter_threads): consolidate corpus extraction into one script; gallery-dl downloads media Replaced convert_cookies.py + run_corpus.py + dedupe_corpus.py (+ the scratch merge_corpus.py) with a single extract_corpus.py that: converts cookies, fetches each URL's full conversation, MERGES overlapping conversations into one thread (union of posts, no duplicate threads/media), downloads ALL media (images + video/mp4) via gallery-dl itself (plain urllib 403'd on pbs.twimg.com and never fetched videos), and renders. Result: 4 merged threads, 33 media files (incl mp4), 0 missing links. cookies_netscape.txt gitignored. --- .gitignore | 1 + .../convert_cookies.py | 41 ------ .../dedupe_corpus.py | 100 ------------- .../extract_corpus.py | 138 ++++++++++++++++++ .../run_corpus.py | 66 --------- 5 files changed, 139 insertions(+), 207 deletions(-) delete mode 100644 conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py delete mode 100644 conductor/tracks/twitter_threads_extraction_20260705/dedupe_corpus.py create mode 100644 conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py delete mode 100644 conductor/tracks/twitter_threads_extraction_20260705/run_corpus.py diff --git a/.gitignore b/.gitignore index e17030a9..fac51778 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ conductor/archive/analysis/video_analysis_*/artifacts/*.vtt # video.log intentionally committed (small text, useful for debugging) conductor/archive/analysis/video_analysis_deob_warmup_20260621/samples scripts/twitter_threads/cookies.txt +conductor/tracks/twitter_threads_extraction_20260705/cookies_netscape.txt diff --git a/conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py b/conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py deleted file mode 100644 index 17b97109..00000000 --- a/conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Convert a JSON cookie export (Cookie-Editor style) to Netscape cookies.txt for gallery-dl. - -Part of the twitter_threads_extraction_20260705 track. Reads the user's JSON cookie -export at scripts/twitter_threads/cookies.txt (gitignored) and writes a Netscape-format -file to a gitignored location so the secret values never land in git. Prints no cookie values. - -Usage: uv run python conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py -""" -from __future__ import annotations - -import json -from pathlib import Path - -SRC = Path("scripts/twitter_threads/cookies.txt") -DST = Path("tests/artifacts/tier2_state/twitter_threads_extraction_20260705/cookies_netscape.txt") - - -def main() -> int: - data = json.loads(SRC.read_text(encoding="utf-8-sig")) - DST.parent.mkdir(parents=True, exist_ok=True) - out: list[str] = ["# Netscape HTTP Cookie File", "# converted from JSON export for gallery-dl", ""] - for c in data: - domain = c.get("domain", "") - include_sub = "TRUE" if domain.startswith(".") else "FALSE" - path = c.get("path", "/") or "/" - secure = "TRUE" if c.get("secure") else "FALSE" - exp = c.get("expirationDate") or c.get("expiry") or c.get("expires") or 2147483647 - try: - exp = int(float(exp)) - except (TypeError, ValueError): - exp = 2147483647 - name = c.get("name", "") - value = c.get("value", "") - out.append("\t".join([domain, include_sub, path, secure, str(exp), name, value])) - DST.write_text("\n".join(out) + "\n", encoding="utf-8") - print(f"wrote {DST} with {len(data)} cookies (values not shown)") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/conductor/tracks/twitter_threads_extraction_20260705/dedupe_corpus.py b/conductor/tracks/twitter_threads_extraction_20260705/dedupe_corpus.py deleted file mode 100644 index d10135dd..00000000 --- a/conductor/tracks/twitter_threads_extraction_20260705/dedupe_corpus.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Deduplicate the docs/twitter corpus: keep the deepest thread per conversation, delete -shallower duplicates, and KEEP + MARK unique branches. Part of the twitter_threads_extraction_20260705 track. - -A "conversation group" = extracted threads whose post-id sets overlap. Within a group the -PRIMARY is the extraction with the most posts (deepest). For every other member: - - if its target tweet is contained in the PRIMARY's post set -> DUPLICATE (delete its dir) - - else -> UNIQUE BRANCH (keep; mark branch_of) - -Branch tracking: kept branches get `branch_of` written into their thread_data.json, and every -kept thread is recorded in docs/twitter/threads_index.json. - -Usage: - uv run python conductor/tracks/twitter_threads_extraction_20260705/dedupe_corpus.py # dry run - uv run python conductor/tracks/twitter_threads_extraction_20260705/dedupe_corpus.py --apply # execute -""" -from __future__ import annotations - -import json -import shutil -import sys -from pathlib import Path - -CORPUS = Path("docs/twitter") - - -def _load() -> dict[str, dict]: - info: dict[str, dict] = {} - for d in sorted(CORPUS.glob("*")): - j = d / "thread_data.json" - if not j.exists(): - continue - t = json.loads(j.read_text(encoding="utf-8")) - tid = t["root_post_id"] - posts = {p["post_id"] for p in t["posts"]} - handle = next((p["handle"] for p in t["posts"] if p["post_id"] == tid), "?") - info[tid] = {"dir": d, "posts": posts, "n": len(posts), "handle": handle, "url": t.get("source_url", ""), "json": j} - return info - - -def _groups(info: dict[str, dict]) -> list[list[str]]: - ids = list(info) - parent = {i: i for i in ids} - def find(x: str) -> str: - while parent[x] != x: - parent[x] = parent[parent[x]] - x = parent[x] - return x - for i in range(len(ids)): - for k in range(i + 1, len(ids)): - a, b = ids[i], ids[k] - if info[a]["posts"] & info[b]["posts"]: - parent[find(a)] = find(b) - out: dict[str, list[str]] = {} - for i in ids: - out.setdefault(find(i), []).append(i) - return list(out.values()) - - -def main() -> int: - apply = "--apply" in sys.argv - info = _load() - if not info: - print("no corpus found under docs/twitter/") - return 1 - index: list[dict] = [] - to_delete: list[str] = [] - branches: list[tuple[str, str]] = [] - for members in _groups(info): - members.sort(key=lambda m: (info[m]["n"], int(m)), reverse=True) - primary = members[0] - print(f"\nGROUP @{info[primary]['handle']} size={len(members)} primary={primary} (n={info[primary]['n']})") - for m in members: - if m == primary: - role = "PRIMARY(deepest)" - index.append({"target": m, "handle": info[m]["handle"], "posts": info[m]["n"], "role": "primary", "branch_of": None, "url": info[m]["url"]}) - elif m in info[primary]["posts"]: - role = "DUPLICATE -> delete" - to_delete.append(m) - else: - role = f"UNIQUE BRANCH of {primary} -> keep + mark" - branches.append((m, primary)) - index.append({"target": m, "handle": info[m]["handle"], "posts": info[m]["n"], "role": "branch", "branch_of": primary, "url": info[m]["url"]}) - print(f" {m} n={info[m]['n']} @{info[m]['handle']} -> {role}") - print(f"\nPLAN: keep {len(index)} threads, delete {len(to_delete)} duplicates, mark {len(branches)} branches. apply={apply}") - if apply: - for m, primary in branches: - j = info[m]["json"] - t = json.loads(j.read_text(encoding="utf-8")) - t["branch_of"] = primary - j.write_text(json.dumps(t, indent=2, ensure_ascii=False), encoding="utf-8") - for m in to_delete: - shutil.rmtree(info[m]["dir"], ignore_errors=True) - print(f"deleted {info[m]['dir']}") - (CORPUS / "threads_index.json").write_text(json.dumps(sorted(index, key=lambda e: e["target"]), indent=2, ensure_ascii=False), encoding="utf-8") - print(f"wrote {CORPUS / 'threads_index.json'}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py b/conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py new file mode 100644 index 00000000..8bbea57e --- /dev/null +++ b/conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py @@ -0,0 +1,138 @@ +"""One-shot corpus extractor for the twitter_threads_extraction_20260705 track. + +Does the whole job in one place (replaces the earlier convert_cookies/run_corpus/merge scripts): + 1. converts the user's JSON cookie export -> Netscape cookies (gitignored) + 2. fetches each URL's FULL conversation via gallery-dl (structure/text) + 3. MERGES overlapping conversations into one thread each (union of posts, no duplicates) + 4. downloads ALL media (images + video/mp4) with gallery-dl itself, which handles X.com + auth / 403 / GIF->mp4 muxing (plain urllib gets 403 and misses videos) + 5. renders one thread.md per merged conversation, linking the downloaded media + 6. writes docs/twitter/threads_index.json + +Output: docs/twitter//{thread.md, thread_data.json, media/} + +Usage: + uv run python conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py +""" +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from scripts.twitter_threads.error_types import ThreadData +from scripts.twitter_threads.fetch_thread import fetch_thread_from_url, thread_to_dict +from scripts.twitter_threads.render_markdown import render_markdown + +TRACK = Path(__file__).resolve().parent +COOKIES_JSON = Path("scripts/twitter_threads/cookies.txt") +COOKIES_NS = TRACK / "cookies_netscape.txt" +CORPUS = Path("docs/twitter") + +URLS = [ + "https://x.com/NOTimothyLottes/status/1757198624818168210", + "https://x.com/NOTimothyLottes/status/1653570742762479620", + "https://x.com/NOTimothyLottes/status/1917646466417381426", + "https://x.com/NOTimothyLottes/status/1917645859791200562", + "https://x.com/NOTimothyLottes/status/1917644904055910502", + "https://x.com/NOTimothyLottes/status/1917642786804785230", + "https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20", + "https://x.com/VPCOMPRESSB/status/1987744335333622188?s=20", +] + + +def ensure_cookies() -> None: + data = json.loads(COOKIES_JSON.read_text(encoding="utf-8-sig")) + out = ["# Netscape HTTP Cookie File", ""] + for c in data: + domain = c.get("domain", "") + inc = "TRUE" if domain.startswith(".") else "FALSE" + path = c.get("path", "/") or "/" + secure = "TRUE" if c.get("secure") else "FALSE" + exp = c.get("expirationDate") or c.get("expiry") or 2147483647 + try: + exp = int(float(exp)) + except (TypeError, ValueError): + exp = 2147483647 + out.append("\t".join([domain, inc, path, secure, str(exp), c.get("name", ""), c.get("value", "")])) + COOKIES_NS.write_text("\n".join(out) + "\n", encoding="utf-8") + + +def gdl_download(url: str, media_dir: Path) -> None: + media_dir.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["gallery-dl", "--cookies", str(COOKIES_NS), "-o", "text-tweets=true", "-o", "conversations=true", "-D", str(media_dir), url], + capture_output=True, text=True, + ) + + +def groups(items: dict[str, dict]) -> list[list[str]]: + ids = list(items) + parent = {i: i for i in ids} + def find(x: str) -> str: + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + for i in range(len(ids)): + for k in range(i + 1, len(ids)): + if items[ids[i]]["posts"] & items[ids[k]]["posts"]: + parent[find(ids[i])] = find(ids[k]) + out: dict[str, list[str]] = {} + for i in ids: + out.setdefault(find(i), []).append(i) + return list(out.values()) + + +def main() -> int: + ensure_cookies() + items: dict[str, dict] = {} + for url in URLS: + r = fetch_thread_from_url(url, COOKIES_NS) + if not r.is_ok: + print(f"FAIL fetch {url} -> {r.error.kind}: {r.error.detail[:120]}") + continue + td = r.data + items[td.root_post_id] = {"td": td, "posts": {p.post_id for p in td.posts}, "url": url} + if not items: + print("no threads fetched") + return 1 + shutil.rmtree(CORPUS, ignore_errors=True) + CORPUS.mkdir(parents=True, exist_ok=True) + index: list[dict] = [] + for members in groups(items): + members.sort(key=lambda m: (len(items[m]["posts"]), int(m)), reverse=True) + primary = members[0] + handle = next((p.handle for p in items[primary]["td"].posts if p.post_id == primary), "?") + seen: dict[str, object] = {} + for m in members: + for p in items[m]["td"].posts: + seen.setdefault(p.post_id, p) + union = sorted(seen.values(), key=lambda p: int(p.post_id) if p.post_id.isdigit() else 0) + td = ThreadData(root_post_id=primary, posts=tuple(union), source_url=items[primary]["url"]) + base = CORPUS / primary + media_dir = base / "media" + for m in members: + gdl_download(items[m]["url"], media_dir) + media_names = {p.post_id: sorted(f.name for f in media_dir.glob(f"{p.post_id}_*")) for p in td.posts} + media_names = {k: v for k, v in media_names.items() if v} + n_media = sum(len(v) for v in media_names.values()) + data = thread_to_dict(td) + data["source_urls"] = [items[m]["url"] for m in members] + data["member_targets"] = sorted(members) + base.mkdir(parents=True, exist_ok=True) + (base / "thread_data.json").write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + render_markdown(td, media_names, base / "thread.md") + index.append({"target": primary, "handle": handle, "posts": len(union), "media": n_media, "source_urls": [items[m]["url"] for m in members]}) + print(f"OK @{handle} {primary}: posts={len(union)} media={n_media} (merged {len(members)} url(s)) -> {base / 'thread.md'}") + (CORPUS / "threads_index.json").write_text(json.dumps(sorted(index, key=lambda e: e["target"]), indent=2, ensure_ascii=False), encoding="utf-8") + print(f"\nCORPUS: {len(index)} merged threads, {sum(e['media'] for e in index)} media files into {CORPUS}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/conductor/tracks/twitter_threads_extraction_20260705/run_corpus.py b/conductor/tracks/twitter_threads_extraction_20260705/run_corpus.py deleted file mode 100644 index ed81537c..00000000 --- a/conductor/tracks/twitter_threads_extraction_20260705/run_corpus.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Run the twitter_threads pipeline over the 8-URL corpus (Task 5.3) into ./docs/twitter/. - -Part of the twitter_threads_extraction_20260705 track. Uses the Netscape cookies produced -by convert_cookies.py. Output layout per URL: docs/twitter//{thread.md, thread_data.json, media/}. -Prints a per-URL summary; never prints cookie values. - -Usage: - uv run python conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py - uv run python conductor/tracks/twitter_threads_extraction_20260705/run_corpus.py -""" -from __future__ import annotations - -import json -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[3])) - -from scripts.twitter_threads.fetch_thread import fetch_thread_from_url, thread_to_dict -from scripts.twitter_threads.download_media import download_media, media_names_for_post -from scripts.twitter_threads.render_markdown import render_markdown - -COOKIES = Path("tests/artifacts/tier2_state/twitter_threads_extraction_20260705/cookies_netscape.txt") -CORPUS = Path("docs/twitter") - -URLS = [ - "https://x.com/NOTimothyLottes/status/1757198624818168210", - "https://x.com/NOTimothyLottes/status/1653570742762479620", - "https://x.com/NOTimothyLottes/status/1917646466417381426", - "https://x.com/NOTimothyLottes/status/1917645859791200562", - "https://x.com/NOTimothyLottes/status/1917644904055910502", - "https://x.com/NOTimothyLottes/status/1917642786804785230", - "https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20", - "https://x.com/VPCOMPRESSB/status/1987744335333622188?s=20", -] - - -def main() -> int: - CORPUS.mkdir(parents=True, exist_ok=True) - ok = 0 - for url in URLS: - fr = fetch_thread_from_url(url, COOKIES) - if not fr.is_ok: - print(f"FAIL fetch {url} -> {fr.error.kind}: {fr.error.detail[:160]}") - continue - td = fr.data - base = CORPUS / td.root_post_id - media_dir = base / "media" - dr = download_media(td, media_dir) - n_media = len(dr.data) if dr.is_ok else -1 - if not dr.is_ok: - print(f"WARN media {td.root_post_id} -> {dr.error.kind}: {dr.error.detail[:120]}") - (base / "thread_data.json").write_text(json.dumps(thread_to_dict(td), indent=2, ensure_ascii=False), encoding="utf-8") - media_names = {p.post_id: media_names_for_post(p) for p in td.posts if p.media_urls} - rr = render_markdown(td, media_names, base / "thread.md") - handle = next((p.handle for p in td.posts if p.post_id == td.root_post_id), td.posts[0].handle if td.posts else "?") - status = "OK" if rr.is_ok else f"RENDER_FAIL {rr.error.kind}" - print(f"{status} {td.root_post_id}: posts={len(td.posts)} media={n_media} @{handle} -> {base / 'thread.md'}") - if rr.is_ok: - ok += 1 - print(f"\nCORPUS DONE: {ok}/{len(URLS)} threads rendered into {CORPUS}") - return 0 if ok == len(URLS) else 1 - - -if __name__ == "__main__": - raise SystemExit(main())