feat(twitter_threads): gallery-dl media download + inline media embeds in Markdown

download_media now shells out to gallery-dl (handles X.com auth/403/video->mp4) instead of urllib, returns the downloaded files, + media_files_for_post glob helper + --cookies CLI. render_markdown embeds images with ![](...) and videos with an HTML <video controls> tag (were plain links). extract_corpus uses the package download_media (one download path). Tests rewritten for the gallery-dl API. Corpus: 4 threads, 33 media (32 img embeds + 1 video), 0 missing.
This commit is contained in:
ed
2026-07-05 19:08:43 -04:00
parent 3f5a2b0659
commit 2c680e23e4
6 changed files with 92 additions and 133 deletions
+8 -6
View File
@@ -3,13 +3,13 @@ from __future__ import annotations
import json
from pathlib import Path
from unittest.mock import patch
from unittest.mock import patch, MagicMock
import pytest
from scripts.twitter_threads.error_types import PostMetrics, PostData, ThreadData, thread_from_dict
from scripts.twitter_threads.fetch_thread import fetch_thread_from_html, thread_to_dict
from scripts.twitter_threads.download_media import media_names_for_post, download_media, main as download_main
from scripts.twitter_threads.download_media import download_media, main as download_main
from scripts.twitter_threads.render_markdown import render_markdown, main as render_main
@@ -59,7 +59,7 @@ def test_pipeline_html_to_markdown(tmp_path: Path) -> None:
assert res.is_ok
td = res.data
assert td is not None
media_names = {p.post_id: media_names_for_post(p) for p in td.posts if p.media_urls}
media_names = {"10": ["10_img1.jpg"]}
out = tmp_path / "thread.md"
rr = render_markdown(td, media_names, out)
assert rr.is_ok
@@ -67,7 +67,7 @@ def test_pipeline_html_to_markdown(tmp_path: Path) -> None:
assert "## Post 1" in text
assert "## Post 2" in text
assert "reply to Post 1" in text
assert "[Media 1](./media/10_img1.jpg)" in text
assert "![Media 1](./media/10_img1.jpg)" in text
def test_download_cli(tmp_path: Path) -> None:
@@ -76,10 +76,12 @@ def test_download_cli(tmp_path: Path) -> None:
thread_data_json = tmp_path / "thread_data.json"
thread_data_json.write_text(json.dumps(thread_to_dict(t)), encoding="utf-8")
mediadir = tmp_path / "media"
with patch("scripts.twitter_threads.download_media.urlopen", return_value=_FakeResp(b"x")):
mediadir.mkdir()
(mediadir / "5_1.jpg").write_bytes(b"x")
with patch("scripts.twitter_threads.download_media.subprocess.run", MagicMock(return_value=MagicMock(returncode=0))):
code = download_main(["--input", str(thread_data_json), "--output", str(mediadir)])
assert code == 0
assert (mediadir / "5_img1.jpg").exists()
assert (mediadir / "5_1.jpg").exists()
def test_render_cli(tmp_path: Path) -> None: