feat(twitter_threads): download_media.py — stdlib urllib + idempotent + typed per-kind naming
media_names_for_post derives <post_id>_<kind><index>.<ext> (img/vid/gif, per-kind counters) from media_urls; download_media(thread, output_dir) -> Result[list[Path]] downloads via stdlib urlopen, idempotent skip when target exists with size>0, URLError -> Result.err(HttpError). Authorized HTTP-boundary mock tests. TDD: red(no module) -> green(6 media + 16 regression pass).
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
"""Download media URLs from ThreadData via stdlib urllib into a local directory."""
|
||||
from __future__ import annotations
|
||||
from pathlib import Path
|
||||
from urllib.request import urlopen
|
||||
from urllib.error import URLError
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
from scripts.twitter_threads.error_types import ErrorInfo, make_error, Result, PostData, ThreadData
|
||||
|
||||
def _classify(url: str) -> tuple[str, str]:
|
||||
parsed = urlparse(url)
|
||||
path = parsed.path
|
||||
if "video.twimg.com" in parsed.netloc or path.endswith(".mp4"):
|
||||
if "tweet_video" in path:
|
||||
return ("gif", "mp4")
|
||||
return ("vid", "mp4")
|
||||
qs = parse_qs(parsed.query)
|
||||
fmt = qs.get("format", [""])[0]
|
||||
if fmt:
|
||||
return ("img", fmt)
|
||||
if "." in path:
|
||||
return ("img", path.rsplit(".", 1)[-1])
|
||||
return ("img", "jpg")
|
||||
|
||||
def media_names_for_post(post: PostData) -> list[str]:
|
||||
counters: dict[str, int] = {}
|
||||
names: list[str] = []
|
||||
for url in post.media_urls:
|
||||
kind, ext = _classify(url)
|
||||
counters[kind] = counters.get(kind, 0) + 1
|
||||
names.append(f"{post.post_id}_{kind}{counters[kind]}.{ext}")
|
||||
return names
|
||||
|
||||
def download_media(thread: ThreadData, output_dir: Path) -> Result[list[Path]]:
|
||||
try:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
return Result.err(make_error("WriteError", "download_media", str(e)))
|
||||
paths: list[Path] = []
|
||||
for post in thread.posts:
|
||||
names = media_names_for_post(post)
|
||||
for url, name in zip(post.media_urls, names):
|
||||
target = output_dir / name
|
||||
if target.exists() and target.stat().st_size > 0:
|
||||
paths.append(target)
|
||||
continue
|
||||
try:
|
||||
with urlopen(url) as resp:
|
||||
data = resp.read()
|
||||
except URLError as e:
|
||||
return Result.err(make_error("HttpError", "download_media", str(e)))
|
||||
try:
|
||||
target.write_bytes(data)
|
||||
except OSError as e:
|
||||
return Result.err(make_error("WriteError", "download_media", str(e)))
|
||||
paths.append(target)
|
||||
return Result.ok(paths)
|
||||
Reference in New Issue
Block a user