feat(twitter_threads): gallery-dl media download + inline media embeds in Markdown

download_media now shells out to gallery-dl (handles X.com auth/403/video->mp4) instead of urllib, returns the downloaded files, + media_files_for_post glob helper + --cookies CLI. render_markdown embeds images with ![](...) and videos with an HTML <video controls> tag (were plain links). extract_corpus uses the package download_media (one download path). Tests rewritten for the gallery-dl API. Corpus: 4 threads, 33 media (32 img embeds + 1 video), 0 missing.

scripts/twitter_threads/download_media.py

@@ -1,70 +1,44 @@
-"""Download media URLs from ThreadData via stdlib urllib into a local directory."""
+"""Download a Twitter/X thread's media via gallery-dl (handles auth, 403, and video/GIF -> mp4)."""
 from __future__ import annotations
 import argparse
 import json
+import subprocess
 import sys
 from pathlib import Path
-from urllib.request import urlopen
-from urllib.error import URLError
-from urllib.parse import urlparse, parse_qs
 try:
- from scripts.twitter_threads.error_types import ErrorInfo, make_error, Result, PostData, ThreadData, thread_from_dict
+ from scripts.twitter_threads.error_types import ErrorInfo, make_error, Result, ThreadData, thread_from_dict
 except ImportError:
- from error_types import ErrorInfo, make_error, Result, PostData, ThreadData, thread_from_dict
+ from error_types import ErrorInfo, make_error, Result, ThreadData, thread_from_dict
 
-def _classify(url: str) -> tuple[str, str]:
- parsed = urlparse(url)
- path = parsed.path
- if "video.twimg.com" in parsed.netloc or path.endswith(".mp4"):
-  if "tweet_video" in path:
-   return ("gif", "mp4")
-  return ("vid", "mp4")
- qs = parse_qs(parsed.query)
- fmt = qs.get("format", [""])[0]
- if fmt:
-  return ("img", fmt)
- if "." in path:
-  return ("img", path.rsplit(".", 1)[-1])
- return ("img", "jpg")
 
-def media_names_for_post(post: PostData) -> list[str]:
- counters: dict[str, int] = {}
- names: list[str] = []
- for url in post.media_urls:
-  kind, ext = _classify(url)
-  counters[kind] = counters.get(kind, 0) + 1
-  names.append(f"{post.post_id}_{kind}{counters[kind]}.{ext}")
- return names
 
-def download_media(thread: ThreadData, output_dir: Path) -> Result[list[Path]]:
+def download_media(thread: ThreadData, output_dir: Path, cookies_path: Path | None = None) -> Result[list[Path]]:
  try:
   output_dir.mkdir(parents=True, exist_ok=True)
  except OSError as e:
   return Result.err(make_error("WriteError", "download_media", str(e)))
- paths: list[Path] = []
- for post in thread.posts:
-  names = media_names_for_post(post)
-  for url, name in zip(post.media_urls, names):
-   target = output_dir / name
-   if target.exists() and target.stat().st_size > 0:
-    paths.append(target)
-    continue
-   try:
-    with urlopen(url) as resp:
-     data = resp.read()
-   except URLError as e:
-    return Result.err(make_error("HttpError", "download_media", str(e)))
-   try:
-    target.write_bytes(data)
-   except OSError as e:
-    return Result.err(make_error("WriteError", "download_media", str(e)))
-   paths.append(target)
- return Result.ok(paths)
+ args = ["gallery-dl", "-o", "text-tweets=true", "-o", "conversations=true", "-D", str(output_dir)]
+ if cookies_path is not None:
+  args += ["--cookies", str(cookies_path)]
+ args.append(thread.source_url)
+ try:
+  completed = subprocess.run(args, capture_output=True, text=True)
+ except OSError as e:
+  return Result.err(make_error("GalleryDlError", "download_media", str(e)))
+ if completed.returncode != 0:
+  return Result.err(make_error("GalleryDlError", "download_media", completed.stderr[:500]))
+ post_ids = {p.post_id for p in thread.posts}
+ files = [f for f in output_dir.rglob("*") if f.is_file() and f.name.split("_")[0] in post_ids]
+ return Result.ok(sorted(files, key=lambda f: f.name))
+
+def media_files_for_post(output_dir: Path, post_id: str) -> list[str]:
+ return sorted(f.name for f in output_dir.glob(f"{post_id}_*") if f.is_file())
 
 def main(argv: list[str] | None = None) -> int:
- parser = argparse.ArgumentParser(description="Download media for a Twitter/X thread_data.json.")
+ parser = argparse.ArgumentParser(description="Download media for a Twitter/X thread_data.json via gallery-dl.")
  parser.add_argument("--input", type=Path, required=True, help="thread_data.json path")
  parser.add_argument("--output", type=Path, required=True, help="media output directory")
+ parser.add_argument("--cookies", type=Path, default=None, help="cookies.txt for gallery-dl auth")
  args = parser.parse_args(argv)
  try:
   raw = args.input.read_text(encoding="utf-8")
@@ -77,7 +51,7 @@ def main(argv: list[str] | None = None) -> int:
   sys.stderr.write(f"error: JsonParseError: {e}\n")
   return 1
  thread = thread_from_dict(data)
- result = download_media(thread, args.output)
+ result = download_media(thread, args.output, args.cookies)
  if not result.is_ok:
   sys.stderr.write(f"error: {result.error.kind}: {result.error.detail}\n")
   return 1

scripts/twitter_threads/render_markdown.py

@@ -43,6 +43,14 @@ def _render_yaml(thread: ThreadData, title: str) -> list[str]:
   "---",
  ]
 
+def _media_line(index: int, name: str) -> str:
+ ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
+ if ext in ("mp4", "mov", "webm", "m4v"):
+  return f'<video controls src="./media/{name}"></video>'
+ if ext in ("jpg", "jpeg", "png", "gif", "webp", "bmp"):
+  return f"![Media {index}](./media/{name})"
+ return f"[Media {index}](./media/{name})"
+
 def _render_post(post: PostData, n: int, id_to_index: dict[str, int], media_names: dict[str, list[str]]) -> list[str]:
  lines: list[str] = []
  header = f"## Post {n} ({post.timestamp})"
@@ -58,7 +66,7 @@ def _render_post(post: PostData, n: int, id_to_index: dict[str, int], media_name
  names = media_names.get(post.post_id, [])
  if names:
   for k, name in enumerate(names, start=1):
-   lines.append(f"[Media {k}](./media/{name})")
+   lines.append(_media_line(k, name))
   lines.append("")
  return lines