feat(twitter_threads): full-thread extraction (conversations) + front-matter from target + corpus dedupe
fetch_thread_from_url now passes -o conversations=true (full threads, not single tweets), sorts posts chronologically, and sets root_post_id to the URL's target tweet. render_markdown builds front-matter from the target (root_post_id) post, not the earliest conversation tweet (fixes wrong @handle). Added dedupe_corpus.py: keeps deepest thread per conversation, deletes duplicates, keeps+marks unique branches (branch_of) + writes threads_index.json. 33 tests pass.
This commit is contained in:
@@ -19,8 +19,14 @@ def _format_view_count(value: int | None) -> str:
|
||||
return "null"
|
||||
return str(value)
|
||||
|
||||
def _root_post(thread: ThreadData) -> PostData:
|
||||
for p in thread.posts:
|
||||
if p.post_id == thread.root_post_id:
|
||||
return p
|
||||
return thread.posts[0]
|
||||
|
||||
def _render_yaml(thread: ThreadData, title: str) -> list[str]:
|
||||
root = thread.posts[0]
|
||||
root = _root_post(thread)
|
||||
return [
|
||||
"---",
|
||||
f'title: "{title}"',
|
||||
@@ -59,7 +65,7 @@ def _render_post(post: PostData, n: int, id_to_index: dict[str, int], media_name
|
||||
def render_markdown(thread: ThreadData, media_names: dict[str, list[str]], output: Path) -> Result[Path]:
|
||||
lines: list[str] = []
|
||||
id_to_index = _build_id_to_index(thread)
|
||||
root = thread.posts[0]
|
||||
root = _root_post(thread)
|
||||
first_line = root.text.splitlines()[0] if root.text.splitlines() else ""
|
||||
title = first_line[:80]
|
||||
lines.extend(_render_yaml(thread, title))
|
||||
|
||||
Reference in New Issue
Block a user