prompted walker from scratch.
This commit is contained in:
1 parent
65b7e566ed
commit
84312401bd
7 files changed
+1869
-2
No files matched your search
@@ -0,0 +1,547 @@
|
||||
"""Recursive cursor walker for X conversation threads.
|
||||
|
||||
``gallery-dl`` with ``conversations=true`` follows only the top-level
|
||||
``cursor-bottom-*`` continuation. Replies reachable only through a
|
||||
``cursor-showmore-*`` nested inside a ``conversationthread-*`` module are
|
||||
dropped without warning. This module issues the same ``TweetDetail``
|
||||
request gallery-dl issues, then follows **every** cursor it finds at any
|
||||
depth.
|
||||
|
||||
The request shape is read from the installed gallery-dl at runtime
|
||||
rather than hardcoded, so a gallery-dl upgrade is picked up without a
|
||||
code change. When gallery-dl does not expose the endpoint, this module
|
||||
raises instead of falling back to a stale literal.
|
||||
|
||||
Nothing here writes to the vault.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import textwrap
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from .paths import COOKIES
|
||||
except ImportError: # pragma: no cover - script-mode import
|
||||
from paths import COOKIES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# gallery-dl is imported lazily: only the walker needs it, and importing
|
||||
# it costs ~100 ms at module import.
|
||||
_gallery_dl: Any = None
|
||||
|
||||
# Marker standing in for gallery-dl's ``tweet_id`` argument in the
|
||||
# variables template it reads from its own source.
|
||||
FOCAL_TWEET_ID = "__focal_tweet_id__"
|
||||
|
||||
|
||||
class WalkerError(RuntimeError):
|
||||
"""Raised when the walker cannot build or run a request."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WalkerResult:
|
||||
"""What one walk found."""
|
||||
post_ids: tuple[str, ...]
|
||||
calls: int
|
||||
per_focal: dict[str, tuple[str, ...]]
|
||||
stopped_reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Page:
|
||||
"""One TweetDetail response, reduced to what the walker needs."""
|
||||
post_ids: tuple[str, ...]
|
||||
conversation_ids: tuple[str, ...]
|
||||
reply_counts: tuple[int, ...]
|
||||
reply_to_ids: tuple[str | None, ...]
|
||||
cursors: tuple[str, ...]
|
||||
top_level_cursor: str | None
|
||||
|
||||
|
||||
def _load_gallery_dl() -> Any:
|
||||
"""Import gallery-dl's Twitter extractor module."""
|
||||
global _gallery_dl
|
||||
if _gallery_dl is None:
|
||||
try:
|
||||
from gallery_dl.extractor import twitter as gd_twitter
|
||||
except ImportError as exc: # pragma: no cover - optional dependency
|
||||
raise WalkerError(
|
||||
"gallery-dl is required by the walker: pip install gallery-dl"
|
||||
) from exc
|
||||
_gallery_dl = gd_twitter
|
||||
return _gallery_dl
|
||||
|
||||
|
||||
def tweet_detail_endpoint() -> str:
|
||||
"""Return the TweetDetail GraphQL endpoint from the installed gallery-dl.
|
||||
|
||||
Raises ``WalkerError`` when gallery-dl does not expose it. The
|
||||
endpoint carries an operation hash that upstream rotates, so there is
|
||||
deliberately no hardcoded fallback.
|
||||
"""
|
||||
twitter = _load_gallery_dl()
|
||||
try:
|
||||
constants = twitter.TwitterAPI.tweet_detail.__code__.co_consts
|
||||
except AttributeError as exc: # pragma: no cover - upstream drift
|
||||
raise WalkerError(
|
||||
"gallery-dl no longer exposes TwitterAPI.tweet_detail"
|
||||
) from exc
|
||||
for value in constants:
|
||||
if isinstance(value, str) and value.endswith("/TweetDetail"):
|
||||
return value
|
||||
raise WalkerError("gallery-dl does not expose a TweetDetail endpoint")
|
||||
|
||||
|
||||
def _eval_literal(node: ast.AST) -> Any:
|
||||
"""Evaluate a literal AST node, tolerating a bare name for the tweet id.
|
||||
|
||||
gallery-dl writes ``"focalTweetId": tweet_id`` where ``tweet_id`` is
|
||||
the method argument. That name is replaced with the
|
||||
``FOCAL_TWEET_ID`` marker and filled in per request.
|
||||
"""
|
||||
if isinstance(node, ast.Constant):
|
||||
return node.value
|
||||
if isinstance(node, ast.Name):
|
||||
return FOCAL_TWEET_ID
|
||||
if isinstance(node, ast.Dict):
|
||||
return {
|
||||
_eval_literal(key): _eval_literal(value)
|
||||
for key, value in zip(node.keys, node.values)
|
||||
if key is not None
|
||||
}
|
||||
if isinstance(node, ast.List):
|
||||
return [_eval_literal(element) for element in node.elts]
|
||||
if isinstance(node, ast.Tuple):
|
||||
return tuple(_eval_literal(element) for element in node.elts)
|
||||
if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
|
||||
return -_eval_literal(node.operand)
|
||||
raise WalkerError(f"gallery-dl uses a non-literal expression: {type(node).__name__}")
|
||||
|
||||
|
||||
def _literal_from_source(source: str, name: str) -> dict[str, Any]:
|
||||
"""Extract a dict literal assigned to ``name`` from gallery-dl source.
|
||||
|
||||
The variables template and field toggles are literals in
|
||||
``TwitterAPI.tweet_detail``. Reading them from the installed source
|
||||
keeps the walker in step with gallery-dl instead of freezing a copy.
|
||||
"""
|
||||
tree = ast.parse(textwrap.dedent(source))
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Assign):
|
||||
continue
|
||||
for target in node.targets:
|
||||
if isinstance(target, ast.Name) and target.id == name:
|
||||
value = _eval_literal(node.value)
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
raise WalkerError(f"gallery-dl does not define a {name} literal")
|
||||
|
||||
|
||||
def request_shape() -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
"""Return the ``(variables, field_toggles)`` pair gallery-dl sends."""
|
||||
twitter = _load_gallery_dl()
|
||||
source = inspect.getsource(twitter.TwitterAPI.tweet_detail)
|
||||
variables = _literal_from_source(source, "variables")
|
||||
toggles = _literal_from_source(source, "field_toggles")
|
||||
return variables, toggles
|
||||
|
||||
|
||||
def build_extractor() -> Any:
|
||||
"""Build a gallery-dl extractor that is authenticated with the vault cookies.
|
||||
|
||||
Cookies, the CSRF token, and the transaction id all come from
|
||||
gallery-dl's own request layer, so the walker speaks the same
|
||||
protocol gallery-dl does.
|
||||
"""
|
||||
twitter = _load_gallery_dl()
|
||||
url = "https://x.com/i/web/status/1"
|
||||
extractor = twitter.TwitterTweetExtractor.from_url(url)
|
||||
if extractor is None:
|
||||
raise WalkerError("gallery-dl could not build a Twitter tweet extractor")
|
||||
extractor.initialize()
|
||||
extractor.cookies_load(str(COOKIES))
|
||||
csrf = extractor.cookies.get("ct0", domain=".x.com")
|
||||
if csrf:
|
||||
extractor.api.headers["x-csrf-token"] = csrf
|
||||
extractor.api.headers["x-twitter-auth-type"] = "OAuth2Session"
|
||||
extractor.login()
|
||||
return extractor
|
||||
|
||||
|
||||
def _cursor_value(entry: dict[str, Any]) -> str | None:
|
||||
"""Extract a continuation cursor value from an entry.
|
||||
|
||||
Twitter uses several shapes for the same thing: a top-level cursor
|
||||
entry, a cursor nested in a thread module, and a legacy
|
||||
``operation.cursor`` wrapper. Cursor values are opaque and are never
|
||||
logged.
|
||||
"""
|
||||
for wrapper_key in ("content", "item"):
|
||||
wrapper = entry.get(wrapper_key)
|
||||
if not isinstance(wrapper, dict):
|
||||
continue
|
||||
item_content = wrapper.get("itemContent")
|
||||
if isinstance(item_content, dict):
|
||||
value = item_content.get("value")
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
value = wrapper.get("value")
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
operation = wrapper.get("operation")
|
||||
if isinstance(operation, dict):
|
||||
cursor = operation.get("cursor")
|
||||
if isinstance(cursor, dict):
|
||||
value = cursor.get("value")
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
_CURSOR_ENTRY = re.compile(r"cursor-(bottom|top|showmore)")
|
||||
|
||||
|
||||
def _is_cursor_entry(entry_id: str) -> bool:
|
||||
"""True for any continuation cursor entry, at any depth."""
|
||||
if not entry_id:
|
||||
return False
|
||||
return bool(_CURSOR_ENTRY.search(entry_id))
|
||||
|
||||
|
||||
def _tweet_ids(node: Any, found: list[tuple[str, str, int, str | None]]) -> None:
|
||||
"""Collect ``(post_id, conversation_id, reply_count, reply_to)`` tuples."""
|
||||
if isinstance(node, dict):
|
||||
result = node.get("result")
|
||||
if isinstance(result, dict) and result.get("__typename") == "Tweet":
|
||||
legacy = result.get("legacy")
|
||||
legacy = legacy if isinstance(legacy, dict) else {}
|
||||
tweet_id = str(legacy.get("id_str") or result.get("rest_id") or "")
|
||||
if tweet_id:
|
||||
conversation = str(legacy.get("conversation_id_str") or "")
|
||||
reply_count = int(legacy.get("reply_count") or 0)
|
||||
reply_to = legacy.get("in_reply_to_status_id_str")
|
||||
entry = (
|
||||
tweet_id,
|
||||
conversation,
|
||||
reply_count,
|
||||
str(reply_to) if reply_to else None,
|
||||
)
|
||||
if entry not in found:
|
||||
found.append(entry)
|
||||
for value in node.values():
|
||||
_tweet_ids(value, found)
|
||||
elif isinstance(node, list):
|
||||
for value in node:
|
||||
_tweet_ids(value, found)
|
||||
|
||||
|
||||
_THREAD_MODULE_PREFIXES = (
|
||||
"homeConversation-",
|
||||
"profile-conversation-",
|
||||
"conversationthread-",
|
||||
)
|
||||
|
||||
|
||||
def _is_thread_module(entry_id: str) -> bool:
|
||||
"""True for a thread module entry, false for its sub-entries.
|
||||
|
||||
A module entry id is a known prefix followed by digits only. Its
|
||||
sub-entries share the prefix but carry a ``-tweet-`` or
|
||||
``-cursor-`` suffix.
|
||||
"""
|
||||
if not entry_id.startswith(_THREAD_MODULE_PREFIXES):
|
||||
return False
|
||||
suffix = entry_id.split("-", 1)[1] if "-" in entry_id else ""
|
||||
return suffix.isdigit()
|
||||
|
||||
|
||||
def parse_page(body: dict[str, Any]) -> _Page:
|
||||
"""Reduce a TweetDetail response to its post ids and its cursors.
|
||||
|
||||
Every cursor in the tree is collected, including the ones nested
|
||||
inside thread modules and ``ShowMoreThreadsPrompt`` entries. The
|
||||
top-level ``cursor-bottom-*`` is reported separately because that is
|
||||
the only one gallery-dl follows.
|
||||
"""
|
||||
post_ids: list[tuple[str, str]] = []
|
||||
cursors: list[str] = []
|
||||
top_level: str | None = None
|
||||
|
||||
def add_cursor(entry: dict[str, Any], top_level_entry: bool) -> None:
|
||||
nonlocal top_level
|
||||
entry_id = entry.get("entryId")
|
||||
if not isinstance(entry_id, str) or not _is_cursor_entry(entry_id):
|
||||
return
|
||||
value = _cursor_value(entry)
|
||||
if not value:
|
||||
return
|
||||
if value not in cursors:
|
||||
cursors.append(value)
|
||||
if top_level_entry and entry_id.startswith("cursor-bottom-"):
|
||||
if top_level is None:
|
||||
top_level = value
|
||||
|
||||
def walk_entries(entries: Any, depth: int) -> None:
|
||||
if not isinstance(entries, list):
|
||||
return
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
entry_id = entry.get("entryId")
|
||||
entry_id = entry_id if isinstance(entry_id, str) else ""
|
||||
if _is_cursor_entry(entry_id):
|
||||
add_cursor(entry, depth == 0)
|
||||
continue
|
||||
if _is_thread_module(entry_id):
|
||||
content = entry.get("content")
|
||||
items = content.get("items") if isinstance(content, dict) else None
|
||||
walk_entries(items, depth + 1)
|
||||
_find_cursors(content, add_cursor)
|
||||
continue
|
||||
if "showmore" in entry_id.lower():
|
||||
content = entry.get("content")
|
||||
items = content.get("items") if isinstance(content, dict) else None
|
||||
_find_cursors(items, add_cursor)
|
||||
walk_entries(items, depth + 1)
|
||||
_find_cursors(entry, add_cursor)
|
||||
_tweet_ids(entry, post_ids)
|
||||
|
||||
def walk_instructions(instructions: Any) -> None:
|
||||
if not isinstance(instructions, list):
|
||||
return
|
||||
for instruction in instructions:
|
||||
if not isinstance(instruction, dict):
|
||||
continue
|
||||
kind = instruction.get("type")
|
||||
if kind == "TimelineAddEntries":
|
||||
walk_entries(instruction.get("entries"), 0)
|
||||
elif kind == "TimelineAddToModule":
|
||||
walk_entries(instruction.get("moduleItems"), 0)
|
||||
elif kind in ("TimelineReplaceEntry", "TimelinePinEntry"):
|
||||
entry = instruction.get("entry")
|
||||
if isinstance(entry, dict):
|
||||
add_cursor(entry, True)
|
||||
_find_cursors(entry, add_cursor)
|
||||
_tweet_ids(entry, post_ids)
|
||||
|
||||
data = body.get("data")
|
||||
data = data if isinstance(data, dict) else {}
|
||||
conversation = data.get("threaded_conversation_with_injections_v2")
|
||||
conversation = conversation if isinstance(conversation, dict) else {}
|
||||
walk_instructions(conversation.get("instructions"))
|
||||
return _Page(
|
||||
post_ids=tuple(entry[0] for entry in post_ids),
|
||||
conversation_ids=tuple(entry[1] for entry in post_ids),
|
||||
reply_counts=tuple(entry[2] for entry in post_ids),
|
||||
reply_to_ids=tuple(entry[3] for entry in post_ids),
|
||||
cursors=tuple(cursors),
|
||||
top_level_cursor=top_level,
|
||||
)
|
||||
|
||||
|
||||
def _find_cursors(node: Any, add_cursor: Any) -> None:
|
||||
"""Find every cursor anywhere below ``node``.
|
||||
|
||||
This is the part gallery-dl skips: cursors inside a thread module
|
||||
or a ``ShowMoreThreadsPrompt`` are never followed.
|
||||
"""
|
||||
if isinstance(node, dict):
|
||||
entry_id = node.get("entryId")
|
||||
if isinstance(entry_id, str) and _is_cursor_entry(entry_id):
|
||||
add_cursor(node, False)
|
||||
for value in node.values():
|
||||
_find_cursors(value, add_cursor)
|
||||
elif isinstance(node, list):
|
||||
for value in node:
|
||||
_find_cursors(value, add_cursor)
|
||||
|
||||
|
||||
def _request(
|
||||
extractor: Any,
|
||||
focal_id: str,
|
||||
cursor: str | None,
|
||||
variables: dict[str, Any],
|
||||
toggles: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Issue one TweetDetail request through gallery-dl's request layer."""
|
||||
request_variables = dict(variables)
|
||||
request_variables["focalTweetId"] = focal_id
|
||||
if cursor:
|
||||
request_variables["cursor"] = cursor
|
||||
params = {
|
||||
"variables": json.dumps(request_variables, separators=(",", ":"), ensure_ascii=False),
|
||||
"features": json.dumps(
|
||||
extractor.api.features_pagination, separators=(",", ":"), ensure_ascii=False
|
||||
),
|
||||
"fieldToggles": json.dumps(toggles, separators=(",", ":"), ensure_ascii=False),
|
||||
}
|
||||
body = extractor.api._call(tweet_detail_endpoint(), params, method="GET")
|
||||
if not isinstance(body, dict):
|
||||
raise WalkerError("TweetDetail returned a non-object body")
|
||||
return body
|
||||
|
||||
|
||||
def _walk_focal(
|
||||
extractor: Any,
|
||||
focal_id: str,
|
||||
request_cap: int,
|
||||
variables: dict[str, Any],
|
||||
toggles: dict[str, Any],
|
||||
capture_dir: Path | None = None,
|
||||
) -> tuple[set[str], dict[str, int], dict[str, str | None], int, str]:
|
||||
"""Walk every cursor under one focal.
|
||||
|
||||
Returns ``(post_ids, reply_counts, reply_to_ids, calls, reason)``.
|
||||
"""
|
||||
post_ids: set[str] = set()
|
||||
reply_counts: dict[str, int] = {}
|
||||
reply_to: dict[str, str | None] = {}
|
||||
seen_cursors: set[str] = set()
|
||||
pending: deque[str | None] = deque([None])
|
||||
calls = 0
|
||||
conversation: str | None = None
|
||||
while pending and calls < request_cap:
|
||||
cursor = pending.popleft()
|
||||
calls += 1
|
||||
body = _request(extractor, focal_id, cursor, variables, toggles)
|
||||
if capture_dir is not None:
|
||||
_capture_page(capture_dir, calls, body)
|
||||
page = parse_page(body)
|
||||
if conversation is None:
|
||||
conversation = _focal_conversation(page, focal_id)
|
||||
for index, tweet_id in enumerate(page.post_ids):
|
||||
if page.conversation_ids[index] != conversation:
|
||||
continue
|
||||
post_ids.add(tweet_id)
|
||||
reply_counts[tweet_id] = page.reply_counts[index]
|
||||
reply_to[tweet_id] = page.reply_to_ids[index]
|
||||
for value in page.cursors:
|
||||
if value in seen_cursors:
|
||||
continue
|
||||
seen_cursors.add(value)
|
||||
pending.append(value)
|
||||
reason = "request-cap" if pending else "exhausted"
|
||||
return post_ids, reply_counts, reply_to, calls, reason
|
||||
|
||||
|
||||
def _capture_page(capture_dir: Path, ordinal: int, body: dict[str, Any]) -> None:
|
||||
"""Write one response page so a run can be inspected afterwards.
|
||||
|
||||
The page holds tweet text and cursor values. It stays in scratch and
|
||||
is never written to the vault.
|
||||
"""
|
||||
capture_dir.mkdir(parents=True, exist_ok=True)
|
||||
target = capture_dir / f"page-{ordinal:03d}.json"
|
||||
target.write_text(
|
||||
json.dumps(body, indent=1, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _focal_conversation(page: _Page, focal_id: str) -> str:
|
||||
"""Return the conversation id the focal's replies share.
|
||||
|
||||
The focal's own conversation id is the one its replies carry. When
|
||||
the focal is not in the first page, the page's modal conversation
|
||||
id is used.
|
||||
"""
|
||||
pairs = list(zip(page.post_ids, page.conversation_ids))
|
||||
for tweet_id, conversation in pairs:
|
||||
if tweet_id == focal_id and conversation:
|
||||
return conversation
|
||||
counts: dict[str, int] = {}
|
||||
for _tweet_id, conversation in pairs:
|
||||
if conversation:
|
||||
counts[conversation] = counts.get(conversation, 0) + 1
|
||||
if not counts:
|
||||
return ""
|
||||
return max(counts.items(), key=lambda item: item[1])[0]
|
||||
|
||||
|
||||
def _under_captured(
|
||||
reply_counts: dict[str, int],
|
||||
reply_to: dict[str, str | None],
|
||||
) -> list[str]:
|
||||
"""Return posts whose reply count exceeds the children actually seen.
|
||||
|
||||
A post that reports more replies than the walk found is a post with
|
||||
a subtree the walk has not reached yet. Walking it is how the walk
|
||||
expands without the operator naming every focal by hand.
|
||||
"""
|
||||
children: dict[str, int] = {}
|
||||
for parent in reply_to.values():
|
||||
if parent:
|
||||
children[parent] = children.get(parent, 0) + 1
|
||||
return [
|
||||
post_id
|
||||
for post_id, count in reply_counts.items()
|
||||
if count > children.get(post_id, 0)
|
||||
]
|
||||
|
||||
def discover(
|
||||
focal_ids: tuple[str, ...] | list[str],
|
||||
request_cap: int = 20,
|
||||
extractor: Any | None = None,
|
||||
max_focals: int = 40,
|
||||
capture_dir: Path | None = None,
|
||||
) -> WalkerResult:
|
||||
"""Walk each focal's conversation and return every post id found.
|
||||
|
||||
The walk expands its own focal set: a post that reports more
|
||||
replies than were found has a subtree not yet reached, so it is
|
||||
walked too. That is what lets one focal find the whole thread.
|
||||
|
||||
``request_cap`` bounds the requests issued per focal.
|
||||
``max_focals`` bounds how many focals the expansion may add, so a
|
||||
conversation that keeps reporting unreached children cannot run
|
||||
away. Requests are sequential. No cursor value is logged or
|
||||
persisted.
|
||||
"""
|
||||
if not focal_ids:
|
||||
return WalkerResult(post_ids=(), calls=0, per_focal={}, stopped_reason="no-focals")
|
||||
variables, toggles = request_shape()
|
||||
if extractor is None:
|
||||
extractor = build_extractor()
|
||||
all_posts: set[str] = set()
|
||||
per_focal: dict[str, tuple[str, ...]] = {}
|
||||
calls = 0
|
||||
stopped = "exhausted"
|
||||
walked: set[str] = set()
|
||||
queue = list(dict.fromkeys(focal_ids))
|
||||
while queue and len(walked) < max_focals:
|
||||
focal = queue.pop(0)
|
||||
if focal in walked:
|
||||
continue
|
||||
walked.add(focal)
|
||||
posts, reply_counts, reply_to, focal_calls, reason = _walk_focal(
|
||||
extractor, focal, request_cap, variables, toggles, capture_dir
|
||||
)
|
||||
per_focal[focal] = tuple(sorted(posts))
|
||||
all_posts.update(posts)
|
||||
calls += focal_calls
|
||||
stopped = reason
|
||||
if len(walked) >= max_focals:
|
||||
stopped = "focal-cap"
|
||||
break
|
||||
for parent in _under_captured(reply_counts, reply_to):
|
||||
if parent not in walked:
|
||||
queue.append(parent)
|
||||
logger.info(
|
||||
"walker found %d posts across %d focals in %d requests",
|
||||
len(all_posts), len(walked), calls,
|
||||
)
|
||||
return WalkerResult(
|
||||
post_ids=tuple(sorted(all_posts)),
|
||||
calls=calls,
|
||||
per_focal=per_focal,
|
||||
stopped_reason=stopped,
|
||||
)
|
||||
Reference in new issue
Block a user