refactor(twitter_threads): consolidate corpus extraction into one script; gallery-dl downloads media
Replaced convert_cookies.py + run_corpus.py + dedupe_corpus.py (+ the scratch merge_corpus.py) with a single extract_corpus.py that: converts cookies, fetches each URL's full conversation, MERGES overlapping conversations into one thread (union of posts, no duplicate threads/media), downloads ALL media (images + video/mp4) via gallery-dl itself (plain urllib 403'd on pbs.twimg.com and never fetched videos), and renders. Result: 4 merged threads, 33 media files (incl mp4), 0 missing links. cookies_netscape.txt gitignored.
This commit is contained in:
@@ -34,3 +34,4 @@ conductor/archive/analysis/video_analysis_*/artifacts/*.vtt
|
||||
# video.log intentionally committed (small text, useful for debugging)
|
||||
conductor/archive/analysis/video_analysis_deob_warmup_20260621/samples
|
||||
scripts/twitter_threads/cookies.txt
|
||||
conductor/tracks/twitter_threads_extraction_20260705/cookies_netscape.txt
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
"""Convert a JSON cookie export (Cookie-Editor style) to Netscape cookies.txt for gallery-dl.
|
||||
|
||||
Part of the twitter_threads_extraction_20260705 track. Reads the user's JSON cookie
|
||||
export at scripts/twitter_threads/cookies.txt (gitignored) and writes a Netscape-format
|
||||
file to a gitignored location so the secret values never land in git. Prints no cookie values.
|
||||
|
||||
Usage: uv run python conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
SRC = Path("scripts/twitter_threads/cookies.txt")
|
||||
DST = Path("tests/artifacts/tier2_state/twitter_threads_extraction_20260705/cookies_netscape.txt")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
data = json.loads(SRC.read_text(encoding="utf-8-sig"))
|
||||
DST.parent.mkdir(parents=True, exist_ok=True)
|
||||
out: list[str] = ["# Netscape HTTP Cookie File", "# converted from JSON export for gallery-dl", ""]
|
||||
for c in data:
|
||||
domain = c.get("domain", "")
|
||||
include_sub = "TRUE" if domain.startswith(".") else "FALSE"
|
||||
path = c.get("path", "/") or "/"
|
||||
secure = "TRUE" if c.get("secure") else "FALSE"
|
||||
exp = c.get("expirationDate") or c.get("expiry") or c.get("expires") or 2147483647
|
||||
try:
|
||||
exp = int(float(exp))
|
||||
except (TypeError, ValueError):
|
||||
exp = 2147483647
|
||||
name = c.get("name", "")
|
||||
value = c.get("value", "")
|
||||
out.append("\t".join([domain, include_sub, path, secure, str(exp), name, value]))
|
||||
DST.write_text("\n".join(out) + "\n", encoding="utf-8")
|
||||
print(f"wrote {DST} with {len(data)} cookies (values not shown)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,100 +0,0 @@
|
||||
"""Deduplicate the docs/twitter corpus: keep the deepest thread per conversation, delete
|
||||
shallower duplicates, and KEEP + MARK unique branches. Part of the twitter_threads_extraction_20260705 track.
|
||||
|
||||
A "conversation group" = extracted threads whose post-id sets overlap. Within a group the
|
||||
PRIMARY is the extraction with the most posts (deepest). For every other member:
|
||||
- if its target tweet is contained in the PRIMARY's post set -> DUPLICATE (delete its dir)
|
||||
- else -> UNIQUE BRANCH (keep; mark branch_of)
|
||||
|
||||
Branch tracking: kept branches get `branch_of` written into their thread_data.json, and every
|
||||
kept thread is recorded in docs/twitter/threads_index.json.
|
||||
|
||||
Usage:
|
||||
uv run python conductor/tracks/twitter_threads_extraction_20260705/dedupe_corpus.py # dry run
|
||||
uv run python conductor/tracks/twitter_threads_extraction_20260705/dedupe_corpus.py --apply # execute
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
CORPUS = Path("docs/twitter")
|
||||
|
||||
|
||||
def _load() -> dict[str, dict]:
|
||||
info: dict[str, dict] = {}
|
||||
for d in sorted(CORPUS.glob("*")):
|
||||
j = d / "thread_data.json"
|
||||
if not j.exists():
|
||||
continue
|
||||
t = json.loads(j.read_text(encoding="utf-8"))
|
||||
tid = t["root_post_id"]
|
||||
posts = {p["post_id"] for p in t["posts"]}
|
||||
handle = next((p["handle"] for p in t["posts"] if p["post_id"] == tid), "?")
|
||||
info[tid] = {"dir": d, "posts": posts, "n": len(posts), "handle": handle, "url": t.get("source_url", ""), "json": j}
|
||||
return info
|
||||
|
||||
|
||||
def _groups(info: dict[str, dict]) -> list[list[str]]:
|
||||
ids = list(info)
|
||||
parent = {i: i for i in ids}
|
||||
def find(x: str) -> str:
|
||||
while parent[x] != x:
|
||||
parent[x] = parent[parent[x]]
|
||||
x = parent[x]
|
||||
return x
|
||||
for i in range(len(ids)):
|
||||
for k in range(i + 1, len(ids)):
|
||||
a, b = ids[i], ids[k]
|
||||
if info[a]["posts"] & info[b]["posts"]:
|
||||
parent[find(a)] = find(b)
|
||||
out: dict[str, list[str]] = {}
|
||||
for i in ids:
|
||||
out.setdefault(find(i), []).append(i)
|
||||
return list(out.values())
|
||||
|
||||
|
||||
def main() -> int:
|
||||
apply = "--apply" in sys.argv
|
||||
info = _load()
|
||||
if not info:
|
||||
print("no corpus found under docs/twitter/")
|
||||
return 1
|
||||
index: list[dict] = []
|
||||
to_delete: list[str] = []
|
||||
branches: list[tuple[str, str]] = []
|
||||
for members in _groups(info):
|
||||
members.sort(key=lambda m: (info[m]["n"], int(m)), reverse=True)
|
||||
primary = members[0]
|
||||
print(f"\nGROUP @{info[primary]['handle']} size={len(members)} primary={primary} (n={info[primary]['n']})")
|
||||
for m in members:
|
||||
if m == primary:
|
||||
role = "PRIMARY(deepest)"
|
||||
index.append({"target": m, "handle": info[m]["handle"], "posts": info[m]["n"], "role": "primary", "branch_of": None, "url": info[m]["url"]})
|
||||
elif m in info[primary]["posts"]:
|
||||
role = "DUPLICATE -> delete"
|
||||
to_delete.append(m)
|
||||
else:
|
||||
role = f"UNIQUE BRANCH of {primary} -> keep + mark"
|
||||
branches.append((m, primary))
|
||||
index.append({"target": m, "handle": info[m]["handle"], "posts": info[m]["n"], "role": "branch", "branch_of": primary, "url": info[m]["url"]})
|
||||
print(f" {m} n={info[m]['n']} @{info[m]['handle']} -> {role}")
|
||||
print(f"\nPLAN: keep {len(index)} threads, delete {len(to_delete)} duplicates, mark {len(branches)} branches. apply={apply}")
|
||||
if apply:
|
||||
for m, primary in branches:
|
||||
j = info[m]["json"]
|
||||
t = json.loads(j.read_text(encoding="utf-8"))
|
||||
t["branch_of"] = primary
|
||||
j.write_text(json.dumps(t, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
for m in to_delete:
|
||||
shutil.rmtree(info[m]["dir"], ignore_errors=True)
|
||||
print(f"deleted {info[m]['dir']}")
|
||||
(CORPUS / "threads_index.json").write_text(json.dumps(sorted(index, key=lambda e: e["target"]), indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(f"wrote {CORPUS / 'threads_index.json'}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,138 @@
|
||||
"""One-shot corpus extractor for the twitter_threads_extraction_20260705 track.
|
||||
|
||||
Does the whole job in one place (replaces the earlier convert_cookies/run_corpus/merge scripts):
|
||||
1. converts the user's JSON cookie export -> Netscape cookies (gitignored)
|
||||
2. fetches each URL's FULL conversation via gallery-dl (structure/text)
|
||||
3. MERGES overlapping conversations into one thread each (union of posts, no duplicates)
|
||||
4. downloads ALL media (images + video/mp4) with gallery-dl itself, which handles X.com
|
||||
auth / 403 / GIF->mp4 muxing (plain urllib gets 403 and misses videos)
|
||||
5. renders one thread.md per merged conversation, linking the downloaded media
|
||||
6. writes docs/twitter/threads_index.json
|
||||
|
||||
Output: docs/twitter/<deepest_target_id>/{thread.md, thread_data.json, media/}
|
||||
|
||||
Usage:
|
||||
uv run python conductor/tracks/twitter_threads_extraction_20260705/extract_corpus.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
|
||||
|
||||
from scripts.twitter_threads.error_types import ThreadData
|
||||
from scripts.twitter_threads.fetch_thread import fetch_thread_from_url, thread_to_dict
|
||||
from scripts.twitter_threads.render_markdown import render_markdown
|
||||
|
||||
TRACK = Path(__file__).resolve().parent
|
||||
COOKIES_JSON = Path("scripts/twitter_threads/cookies.txt")
|
||||
COOKIES_NS = TRACK / "cookies_netscape.txt"
|
||||
CORPUS = Path("docs/twitter")
|
||||
|
||||
URLS = [
|
||||
"https://x.com/NOTimothyLottes/status/1757198624818168210",
|
||||
"https://x.com/NOTimothyLottes/status/1653570742762479620",
|
||||
"https://x.com/NOTimothyLottes/status/1917646466417381426",
|
||||
"https://x.com/NOTimothyLottes/status/1917645859791200562",
|
||||
"https://x.com/NOTimothyLottes/status/1917644904055910502",
|
||||
"https://x.com/NOTimothyLottes/status/1917642786804785230",
|
||||
"https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20",
|
||||
"https://x.com/VPCOMPRESSB/status/1987744335333622188?s=20",
|
||||
]
|
||||
|
||||
|
||||
def ensure_cookies() -> None:
|
||||
data = json.loads(COOKIES_JSON.read_text(encoding="utf-8-sig"))
|
||||
out = ["# Netscape HTTP Cookie File", ""]
|
||||
for c in data:
|
||||
domain = c.get("domain", "")
|
||||
inc = "TRUE" if domain.startswith(".") else "FALSE"
|
||||
path = c.get("path", "/") or "/"
|
||||
secure = "TRUE" if c.get("secure") else "FALSE"
|
||||
exp = c.get("expirationDate") or c.get("expiry") or 2147483647
|
||||
try:
|
||||
exp = int(float(exp))
|
||||
except (TypeError, ValueError):
|
||||
exp = 2147483647
|
||||
out.append("\t".join([domain, inc, path, secure, str(exp), c.get("name", ""), c.get("value", "")]))
|
||||
COOKIES_NS.write_text("\n".join(out) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def gdl_download(url: str, media_dir: Path) -> None:
|
||||
media_dir.mkdir(parents=True, exist_ok=True)
|
||||
subprocess.run(
|
||||
["gallery-dl", "--cookies", str(COOKIES_NS), "-o", "text-tweets=true", "-o", "conversations=true", "-D", str(media_dir), url],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
|
||||
|
||||
def groups(items: dict[str, dict]) -> list[list[str]]:
|
||||
ids = list(items)
|
||||
parent = {i: i for i in ids}
|
||||
def find(x: str) -> str:
|
||||
while parent[x] != x:
|
||||
parent[x] = parent[parent[x]]
|
||||
x = parent[x]
|
||||
return x
|
||||
for i in range(len(ids)):
|
||||
for k in range(i + 1, len(ids)):
|
||||
if items[ids[i]]["posts"] & items[ids[k]]["posts"]:
|
||||
parent[find(ids[i])] = find(ids[k])
|
||||
out: dict[str, list[str]] = {}
|
||||
for i in ids:
|
||||
out.setdefault(find(i), []).append(i)
|
||||
return list(out.values())
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ensure_cookies()
|
||||
items: dict[str, dict] = {}
|
||||
for url in URLS:
|
||||
r = fetch_thread_from_url(url, COOKIES_NS)
|
||||
if not r.is_ok:
|
||||
print(f"FAIL fetch {url} -> {r.error.kind}: {r.error.detail[:120]}")
|
||||
continue
|
||||
td = r.data
|
||||
items[td.root_post_id] = {"td": td, "posts": {p.post_id for p in td.posts}, "url": url}
|
||||
if not items:
|
||||
print("no threads fetched")
|
||||
return 1
|
||||
shutil.rmtree(CORPUS, ignore_errors=True)
|
||||
CORPUS.mkdir(parents=True, exist_ok=True)
|
||||
index: list[dict] = []
|
||||
for members in groups(items):
|
||||
members.sort(key=lambda m: (len(items[m]["posts"]), int(m)), reverse=True)
|
||||
primary = members[0]
|
||||
handle = next((p.handle for p in items[primary]["td"].posts if p.post_id == primary), "?")
|
||||
seen: dict[str, object] = {}
|
||||
for m in members:
|
||||
for p in items[m]["td"].posts:
|
||||
seen.setdefault(p.post_id, p)
|
||||
union = sorted(seen.values(), key=lambda p: int(p.post_id) if p.post_id.isdigit() else 0)
|
||||
td = ThreadData(root_post_id=primary, posts=tuple(union), source_url=items[primary]["url"])
|
||||
base = CORPUS / primary
|
||||
media_dir = base / "media"
|
||||
for m in members:
|
||||
gdl_download(items[m]["url"], media_dir)
|
||||
media_names = {p.post_id: sorted(f.name for f in media_dir.glob(f"{p.post_id}_*")) for p in td.posts}
|
||||
media_names = {k: v for k, v in media_names.items() if v}
|
||||
n_media = sum(len(v) for v in media_names.values())
|
||||
data = thread_to_dict(td)
|
||||
data["source_urls"] = [items[m]["url"] for m in members]
|
||||
data["member_targets"] = sorted(members)
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
(base / "thread_data.json").write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
render_markdown(td, media_names, base / "thread.md")
|
||||
index.append({"target": primary, "handle": handle, "posts": len(union), "media": n_media, "source_urls": [items[m]["url"] for m in members]})
|
||||
print(f"OK @{handle} {primary}: posts={len(union)} media={n_media} (merged {len(members)} url(s)) -> {base / 'thread.md'}")
|
||||
(CORPUS / "threads_index.json").write_text(json.dumps(sorted(index, key=lambda e: e["target"]), indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(f"\nCORPUS: {len(index)} merged threads, {sum(e['media'] for e in index)} media files into {CORPUS}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,66 +0,0 @@
|
||||
"""Run the twitter_threads pipeline over the 8-URL corpus (Task 5.3) into ./docs/twitter/.
|
||||
|
||||
Part of the twitter_threads_extraction_20260705 track. Uses the Netscape cookies produced
|
||||
by convert_cookies.py. Output layout per URL: docs/twitter/<post_id>/{thread.md, thread_data.json, media/}.
|
||||
Prints a per-URL summary; never prints cookie values.
|
||||
|
||||
Usage:
|
||||
uv run python conductor/tracks/twitter_threads_extraction_20260705/convert_cookies.py
|
||||
uv run python conductor/tracks/twitter_threads_extraction_20260705/run_corpus.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
|
||||
|
||||
from scripts.twitter_threads.fetch_thread import fetch_thread_from_url, thread_to_dict
|
||||
from scripts.twitter_threads.download_media import download_media, media_names_for_post
|
||||
from scripts.twitter_threads.render_markdown import render_markdown
|
||||
|
||||
COOKIES = Path("tests/artifacts/tier2_state/twitter_threads_extraction_20260705/cookies_netscape.txt")
|
||||
CORPUS = Path("docs/twitter")
|
||||
|
||||
URLS = [
|
||||
"https://x.com/NOTimothyLottes/status/1757198624818168210",
|
||||
"https://x.com/NOTimothyLottes/status/1653570742762479620",
|
||||
"https://x.com/NOTimothyLottes/status/1917646466417381426",
|
||||
"https://x.com/NOTimothyLottes/status/1917645859791200562",
|
||||
"https://x.com/NOTimothyLottes/status/1917644904055910502",
|
||||
"https://x.com/NOTimothyLottes/status/1917642786804785230",
|
||||
"https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20",
|
||||
"https://x.com/VPCOMPRESSB/status/1987744335333622188?s=20",
|
||||
]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
CORPUS.mkdir(parents=True, exist_ok=True)
|
||||
ok = 0
|
||||
for url in URLS:
|
||||
fr = fetch_thread_from_url(url, COOKIES)
|
||||
if not fr.is_ok:
|
||||
print(f"FAIL fetch {url} -> {fr.error.kind}: {fr.error.detail[:160]}")
|
||||
continue
|
||||
td = fr.data
|
||||
base = CORPUS / td.root_post_id
|
||||
media_dir = base / "media"
|
||||
dr = download_media(td, media_dir)
|
||||
n_media = len(dr.data) if dr.is_ok else -1
|
||||
if not dr.is_ok:
|
||||
print(f"WARN media {td.root_post_id} -> {dr.error.kind}: {dr.error.detail[:120]}")
|
||||
(base / "thread_data.json").write_text(json.dumps(thread_to_dict(td), indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
media_names = {p.post_id: media_names_for_post(p) for p in td.posts if p.media_urls}
|
||||
rr = render_markdown(td, media_names, base / "thread.md")
|
||||
handle = next((p.handle for p in td.posts if p.post_id == td.root_post_id), td.posts[0].handle if td.posts else "?")
|
||||
status = "OK" if rr.is_ok else f"RENDER_FAIL {rr.error.kind}"
|
||||
print(f"{status} {td.root_post_id}: posts={len(td.posts)} media={n_media} @{handle} -> {base / 'thread.md'}")
|
||||
if rr.is_ok:
|
||||
ok += 1
|
||||
print(f"\nCORPUS DONE: {ok}/{len(URLS)} threads rendered into {CORPUS}")
|
||||
return 0 if ok == len(URLS) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user