ed
06ff9299e2
fetch_thread_from_html parses the data-attr article contract via stdlib html.parser; _normalize_url strips ?s=20 quote-share suffix (spec FR2); fetch_thread_from_url wraps gallery-dl --dump-json (best-effort, manually smoke-tested); dual-import (absolute/relative) makes it runnable via -m AND as a bare script (VC6/G5); CLI writes thread_data.json. TDD: red(no module) -> green(7 fetch + 22 regression = 29 passed).
133 lines
4.6 KiB
Python
133 lines
4.6 KiB
Python
"""RED tests for scripts.twitter_threads.fetch_thread (normalize URL + HTML to ThreadData)."""
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from scripts.twitter_threads.error_types import PostData, ThreadData
|
|
from scripts.twitter_threads.fetch_thread import fetch_thread_from_html, _normalize_url
|
|
|
|
|
|
def test_normalize_url_strips_query() -> None:
|
|
assert _normalize_url("https://x.com/VPCOMPRESSB/status/1991383117571957052?s=20") == "https://x.com/VPCOMPRESSB/status/1991383117571957052"
|
|
assert _normalize_url("https://x.com/NOTimothyLottes/status/123") == "https://x.com/NOTimothyLottes/status/123"
|
|
|
|
|
|
def test_parse_html_single_post(tmp_path: Path) -> None:
|
|
html = """<html><body>
|
|
<link rel="canonical" href="https://x.com/NOTimothyLottes/status/1757198624818168210">
|
|
<article data-post-id="1757198624818168210" data-author="Timothy Lottes" data-handle="NOTimothyLottes" data-reply-count="5" data-repost-count="10" data-like-count="100" data-view-count="5000">
|
|
<time datetime="2024-02-13T12:00:00Z">Feb 13</time>
|
|
<div data-testid="tweetText">Hello thread</div>
|
|
<img src="https://pbs.twimg.com/media/AAA.jpg">
|
|
</article>
|
|
</body></html>"""
|
|
p = tmp_path / "x.html"
|
|
p.write_text(html, encoding="utf-8")
|
|
result = fetch_thread_from_html(p)
|
|
assert result.is_ok
|
|
td = result.data
|
|
assert td is not None
|
|
assert td.root_post_id == "1757198624818168210"
|
|
assert len(td.posts) == 1
|
|
post = td.posts[0]
|
|
assert post.post_id == "1757198624818168210"
|
|
assert post.author == "Timothy Lottes"
|
|
assert post.handle == "NOTimothyLottes"
|
|
assert post.timestamp == "2024-02-13T12:00:00Z"
|
|
assert "Hello thread" in post.text
|
|
assert post.media_urls == ("https://pbs.twimg.com/media/AAA.jpg",)
|
|
assert post.metrics.reply_count == 5
|
|
assert post.metrics.repost_count == 10
|
|
assert post.metrics.like_count == 100
|
|
assert post.metrics.view_count == 5000
|
|
assert td.source_url == "https://x.com/NOTimothyLottes/status/1757198624818168210"
|
|
|
|
|
|
def test_parse_html_thread(tmp_path: Path) -> None:
|
|
html = """<html><body>
|
|
<article data-post-id="1" data-author="A" data-handle="a">
|
|
<time datetime="2024-01-01T00:00:00Z">d1</time>
|
|
<div data-testid="tweetText">p1</div>
|
|
</article>
|
|
<article data-post-id="2" data-author="A" data-handle="a" data-reply-to="1">
|
|
<time datetime="2024-01-02T00:00:00Z">d2</time>
|
|
<div data-testid="tweetText">p2</div>
|
|
</article>
|
|
<article data-post-id="3" data-author="A" data-handle="a" data-reply-to="2">
|
|
<time datetime="2024-01-03T00:00:00Z">d3</time>
|
|
<div data-testid="tweetText">p3</div>
|
|
</article>
|
|
</body></html>"""
|
|
p = tmp_path / "x.html"
|
|
p.write_text(html, encoding="utf-8")
|
|
result = fetch_thread_from_html(p)
|
|
assert result.is_ok
|
|
td = result.data
|
|
assert td is not None
|
|
assert len(td.posts) == 3
|
|
ids = [pt.post_id for pt in td.posts]
|
|
assert ids == ["1", "2", "3"]
|
|
assert td.posts[1].reply_to_id == "1"
|
|
assert td.posts[2].reply_to_id == "2"
|
|
assert td.posts[0].reply_to_id is None
|
|
|
|
|
|
def test_parse_html_no_media(tmp_path: Path) -> None:
|
|
html = """<html><body>
|
|
<article data-post-id="1" data-author="A" data-handle="a">
|
|
<time datetime="2024-01-01T00:00:00Z">d</time>
|
|
<div data-testid="tweetText">no media here</div>
|
|
</article>
|
|
</body></html>"""
|
|
p = tmp_path / "x.html"
|
|
p.write_text(html, encoding="utf-8")
|
|
result = fetch_thread_from_html(p)
|
|
assert result.is_ok
|
|
td = result.data
|
|
assert td is not None
|
|
assert td.posts[0].media_urls == ()
|
|
|
|
|
|
def test_parse_html_quote(tmp_path: Path) -> None:
|
|
html = """<html><body>
|
|
<article data-post-id="1" data-author="A" data-handle="a" data-quote-of="999">
|
|
<time datetime="2024-01-01T00:00:00Z">d</time>
|
|
<div data-testid="tweetText">quoting</div>
|
|
</article>
|
|
</body></html>"""
|
|
p = tmp_path / "x.html"
|
|
p.write_text(html, encoding="utf-8")
|
|
result = fetch_thread_from_html(p)
|
|
assert result.is_ok
|
|
td = result.data
|
|
assert td is not None
|
|
assert td.posts[0].quote_of_id == "999"
|
|
|
|
|
|
def test_parse_html_malformed(tmp_path: Path) -> None:
|
|
html = "<html><body><p>no articles here</p></body></html>"
|
|
p = tmp_path / "x.html"
|
|
p.write_text(html, encoding="utf-8")
|
|
result = fetch_thread_from_html(p)
|
|
assert result.is_ok is False
|
|
assert result.error is not None
|
|
assert result.error.kind == "ParseError"
|
|
|
|
|
|
def test_parse_html_view_count_absent(tmp_path: Path) -> None:
|
|
html = """<html><body>
|
|
<article data-post-id="1" data-author="A" data-handle="a">
|
|
<time datetime="2024-01-01T00:00:00Z">d</time>
|
|
<div data-testid="tweetText">x</div>
|
|
</article>
|
|
</body></html>"""
|
|
p = tmp_path / "x.html"
|
|
p.write_text(html, encoding="utf-8")
|
|
result = fetch_thread_from_html(p)
|
|
assert result.is_ok
|
|
td = result.data
|
|
assert td is not None
|
|
assert td.posts[0].metrics.view_count is None
|
|
assert td.posts[0].metrics.reply_count == 0 |