"""Recursive cursor walker for X conversation threads.

``gallery-dl`` with ``conversations=true`` follows only the top-level
``cursor-bottom-*`` continuation. Replies reachable only through a
``cursor-showmore-*`` nested inside a ``conversationthread-*`` module are
dropped without warning. This module issues the same ``TweetDetail``
request gallery-dl issues, then follows **every** cursor it finds at any
depth.

The request shape is read from the installed gallery-dl at runtime
rather than hardcoded, so a gallery-dl upgrade is picked up without a
code change. When gallery-dl does not expose the endpoint, this module
raises instead of falling back to a stale literal.

Nothing here writes to the vault.
"""
from __future__ import annotations

import ast
import inspect
import json
import logging
import re
import textwrap
from collections import deque
from dataclasses import dataclass
from pathlib import Path
from typing import Any

try:
    from .paths import COOKIES
except ImportError:  # pragma: no cover - script-mode import
    from paths import COOKIES

logger = logging.getLogger(__name__)

# gallery-dl is imported lazily: only the walker needs it, and importing
# it costs ~100 ms at module import.
_gallery_dl: Any = None

# Marker standing in for gallery-dl's ``tweet_id`` argument in the
# variables template it reads from its own source.
FOCAL_TWEET_ID = "__focal_tweet_id__"


class WalkerError(RuntimeError):
    """Raised when the walker cannot build or run a request."""


@dataclass(frozen=True)
class WalkerResult:
    """What one walk found."""
    post_ids: tuple[str, ...]
    calls: int
    per_focal: dict[str, tuple[str, ...]]
    stopped_reason: str


@dataclass(frozen=True)
class _Page:
    """One TweetDetail response, reduced to what the walker needs."""
    post_ids: tuple[str, ...]
    conversation_ids: tuple[str, ...]
    reply_counts: tuple[int, ...]
    reply_to_ids: tuple[str | None, ...]
    cursors: tuple[str, ...]
    top_level_cursor: str | None


def _load_gallery_dl() -> Any:
    """Import gallery-dl's Twitter extractor module."""
    global _gallery_dl
    if _gallery_dl is None:
        try:
            from gallery_dl.extractor import twitter as gd_twitter
        except ImportError as exc:  # pragma: no cover - optional dependency
            raise WalkerError(
                "gallery-dl is required by the walker: pip install gallery-dl"
            ) from exc
        _gallery_dl = gd_twitter
    return _gallery_dl


def tweet_detail_endpoint() -> str:
    """Return the TweetDetail GraphQL endpoint from the installed gallery-dl.

    Raises ``WalkerError`` when gallery-dl does not expose it. The
    endpoint carries an operation hash that upstream rotates, so there is
    deliberately no hardcoded fallback.
    """
    twitter = _load_gallery_dl()
    try:
        constants = twitter.TwitterAPI.tweet_detail.__code__.co_consts
    except AttributeError as exc:  # pragma: no cover - upstream drift
        raise WalkerError(
            "gallery-dl no longer exposes TwitterAPI.tweet_detail"
        ) from exc
    for value in constants:
        if isinstance(value, str) and value.endswith("/TweetDetail"):
            return value
    raise WalkerError("gallery-dl does not expose a TweetDetail endpoint")


def _eval_literal(node: ast.AST) -> Any:
    """Evaluate a literal AST node, tolerating a bare name for the tweet id.

    gallery-dl writes ``"focalTweetId": tweet_id`` where ``tweet_id`` is
    the method argument. That name is replaced with the
    ``FOCAL_TWEET_ID`` marker and filled in per request.
    """
    if isinstance(node, ast.Constant):
        return node.value
    if isinstance(node, ast.Name):
        return FOCAL_TWEET_ID
    if isinstance(node, ast.Dict):
        return {
            _eval_literal(key): _eval_literal(value)
            for key, value in zip(node.keys, node.values)
            if key is not None
        }
    if isinstance(node, ast.List):
        return [_eval_literal(element) for element in node.elts]
    if isinstance(node, ast.Tuple):
        return tuple(_eval_literal(element) for element in node.elts)
    if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
        return -_eval_literal(node.operand)
    raise WalkerError(f"gallery-dl uses a non-literal expression: {type(node).__name__}")


def _literal_from_source(source: str, name: str) -> dict[str, Any]:
    """Extract a dict literal assigned to ``name`` from gallery-dl source.

    The variables template and field toggles are literals in
    ``TwitterAPI.tweet_detail``. Reading them from the installed source
    keeps the walker in step with gallery-dl instead of freezing a copy.
    """
    tree = ast.parse(textwrap.dedent(source))
    for node in ast.walk(tree):
        if not isinstance(node, ast.Assign):
            continue
        for target in node.targets:
            if isinstance(target, ast.Name) and target.id == name:
                value = _eval_literal(node.value)
                if isinstance(value, dict):
                    return value
    raise WalkerError(f"gallery-dl does not define a {name} literal")


def request_shape() -> tuple[dict[str, Any], dict[str, Any]]:
    """Return the ``(variables, field_toggles)`` pair gallery-dl sends."""
    twitter = _load_gallery_dl()
    source = inspect.getsource(twitter.TwitterAPI.tweet_detail)
    variables = _literal_from_source(source, "variables")
    toggles = _literal_from_source(source, "field_toggles")
    return variables, toggles


def build_extractor() -> Any:
    """Build a gallery-dl extractor that is authenticated with the vault cookies.

    Cookies, the CSRF token, and the transaction id all come from
    gallery-dl's own request layer, so the walker speaks the same
    protocol gallery-dl does.
    """
    twitter = _load_gallery_dl()
    url = "https://x.com/i/web/status/1"
    extractor = twitter.TwitterTweetExtractor.from_url(url)
    if extractor is None:
        raise WalkerError("gallery-dl could not build a Twitter tweet extractor")
    extractor.initialize()
    extractor.cookies_load(str(COOKIES))
    csrf = extractor.cookies.get("ct0", domain=".x.com")
    if csrf:
        extractor.api.headers["x-csrf-token"] = csrf
    extractor.api.headers["x-twitter-auth-type"] = "OAuth2Session"
    extractor.login()
    return extractor


def _cursor_value(entry: dict[str, Any]) -> str | None:
    """Extract a continuation cursor value from an entry.

    Twitter uses several shapes for the same thing: a top-level cursor
    entry, a cursor nested in a thread module, and a legacy
    ``operation.cursor`` wrapper. Cursor values are opaque and are never
    logged.
    """
    for wrapper_key in ("content", "item"):
        wrapper = entry.get(wrapper_key)
        if not isinstance(wrapper, dict):
            continue
        item_content = wrapper.get("itemContent")
        if isinstance(item_content, dict):
            value = item_content.get("value")
            if isinstance(value, str) and value:
                return value
        value = wrapper.get("value")
        if isinstance(value, str) and value:
            return value
        operation = wrapper.get("operation")
        if isinstance(operation, dict):
            cursor = operation.get("cursor")
            if isinstance(cursor, dict):
                value = cursor.get("value")
                if isinstance(value, str) and value:
                    return value
    return None


_CURSOR_ENTRY = re.compile(r"cursor-(bottom|top|showmore)")


def _is_cursor_entry(entry_id: str) -> bool:
    """True for any continuation cursor entry, at any depth."""
    if not entry_id:
        return False
    return bool(_CURSOR_ENTRY.search(entry_id))


def _tweet_ids(node: Any, found: list[tuple[str, str, int, str | None]]) -> None:
    """Collect ``(post_id, conversation_id, reply_count, reply_to)`` tuples."""
    if isinstance(node, dict):
        result = node.get("result")
        if isinstance(result, dict) and result.get("__typename") == "Tweet":
            legacy = result.get("legacy")
            legacy = legacy if isinstance(legacy, dict) else {}
            tweet_id = str(legacy.get("id_str") or result.get("rest_id") or "")
            if tweet_id:
                conversation = str(legacy.get("conversation_id_str") or "")
                reply_count = int(legacy.get("reply_count") or 0)
                reply_to = legacy.get("in_reply_to_status_id_str")
                entry = (
                    tweet_id,
                    conversation,
                    reply_count,
                    str(reply_to) if reply_to else None,
                )
                if entry not in found:
                    found.append(entry)
        for value in node.values():
            _tweet_ids(value, found)
    elif isinstance(node, list):
        for value in node:
            _tweet_ids(value, found)


_THREAD_MODULE_PREFIXES = (
    "homeConversation-",
    "profile-conversation-",
    "conversationthread-",
)


def _is_thread_module(entry_id: str) -> bool:
    """True for a thread module entry, false for its sub-entries.

    A module entry id is a known prefix followed by digits only. Its
    sub-entries share the prefix but carry a ``-tweet-`` or
    ``-cursor-`` suffix.
    """
    if not entry_id.startswith(_THREAD_MODULE_PREFIXES):
        return False
    suffix = entry_id.split("-", 1)[1] if "-" in entry_id else ""
    return suffix.isdigit()


def parse_page(body: dict[str, Any]) -> _Page:
    """Reduce a TweetDetail response to its post ids and its cursors.

    Every cursor in the tree is collected, including the ones nested
    inside thread modules and ``ShowMoreThreadsPrompt`` entries. The
    top-level ``cursor-bottom-*`` is reported separately because that is
    the only one gallery-dl follows.
    """
    post_ids: list[tuple[str, str]] = []
    cursors: list[str] = []
    top_level: str | None = None

    def add_cursor(entry: dict[str, Any], top_level_entry: bool) -> None:
        nonlocal top_level
        entry_id = entry.get("entryId")
        if not isinstance(entry_id, str) or not _is_cursor_entry(entry_id):
            return
        value = _cursor_value(entry)
        if not value:
            return
        if value not in cursors:
            cursors.append(value)
        if top_level_entry and entry_id.startswith("cursor-bottom-"):
            if top_level is None:
                top_level = value

    def walk_entries(entries: Any, depth: int) -> None:
        if not isinstance(entries, list):
            return
        for entry in entries:
            if not isinstance(entry, dict):
                continue
            entry_id = entry.get("entryId")
            entry_id = entry_id if isinstance(entry_id, str) else ""
            if _is_cursor_entry(entry_id):
                add_cursor(entry, depth == 0)
                continue
            if _is_thread_module(entry_id):
                content = entry.get("content")
                items = content.get("items") if isinstance(content, dict) else None
                walk_entries(items, depth + 1)
                _find_cursors(content, add_cursor)
                continue
            if "showmore" in entry_id.lower():
                content = entry.get("content")
                items = content.get("items") if isinstance(content, dict) else None
                _find_cursors(items, add_cursor)
                walk_entries(items, depth + 1)
            _find_cursors(entry, add_cursor)
            _tweet_ids(entry, post_ids)

    def walk_instructions(instructions: Any) -> None:
        if not isinstance(instructions, list):
            return
        for instruction in instructions:
            if not isinstance(instruction, dict):
                continue
            kind = instruction.get("type")
            if kind == "TimelineAddEntries":
                walk_entries(instruction.get("entries"), 0)
            elif kind == "TimelineAddToModule":
                walk_entries(instruction.get("moduleItems"), 0)
            elif kind in ("TimelineReplaceEntry", "TimelinePinEntry"):
                entry = instruction.get("entry")
                if isinstance(entry, dict):
                    add_cursor(entry, True)
                    _find_cursors(entry, add_cursor)
                    _tweet_ids(entry, post_ids)

    data = body.get("data")
    data = data if isinstance(data, dict) else {}
    conversation = data.get("threaded_conversation_with_injections_v2")
    conversation = conversation if isinstance(conversation, dict) else {}
    walk_instructions(conversation.get("instructions"))
    return _Page(
        post_ids=tuple(entry[0] for entry in post_ids),
        conversation_ids=tuple(entry[1] for entry in post_ids),
        reply_counts=tuple(entry[2] for entry in post_ids),
        reply_to_ids=tuple(entry[3] for entry in post_ids),
        cursors=tuple(cursors),
        top_level_cursor=top_level,
    )


def _find_cursors(node: Any, add_cursor: Any) -> None:
    """Find every cursor anywhere below ``node``.

    This is the part gallery-dl skips: cursors inside a thread module
    or a ``ShowMoreThreadsPrompt`` are never followed.
    """
    if isinstance(node, dict):
        entry_id = node.get("entryId")
        if isinstance(entry_id, str) and _is_cursor_entry(entry_id):
            add_cursor(node, False)
        for value in node.values():
            _find_cursors(value, add_cursor)
    elif isinstance(node, list):
        for value in node:
            _find_cursors(value, add_cursor)


def _request(
    extractor: Any,
    focal_id: str,
    cursor: str | None,
    variables: dict[str, Any],
    toggles: dict[str, Any],
) -> dict[str, Any]:
    """Issue one TweetDetail request through gallery-dl's request layer."""
    request_variables = dict(variables)
    request_variables["focalTweetId"] = focal_id
    if cursor:
        request_variables["cursor"] = cursor
    params = {
        "variables": json.dumps(request_variables, separators=(",", ":"), ensure_ascii=False),
        "features": json.dumps(
            extractor.api.features_pagination, separators=(",", ":"), ensure_ascii=False
        ),
        "fieldToggles": json.dumps(toggles, separators=(",", ":"), ensure_ascii=False),
    }
    body = extractor.api._call(tweet_detail_endpoint(), params, method="GET")
    if not isinstance(body, dict):
        raise WalkerError("TweetDetail returned a non-object body")
    return body


def _walk_focal(
    extractor: Any,
    focal_id: str,
    request_cap: int,
    variables: dict[str, Any],
    toggles: dict[str, Any],
    capture_dir: Path | None = None,
) -> tuple[set[str], dict[str, int], dict[str, str | None], int, str]:
    """Walk every cursor under one focal.

    Returns ``(post_ids, reply_counts, reply_to_ids, calls, reason)``.
    """
    post_ids: set[str] = set()
    reply_counts: dict[str, int] = {}
    reply_to: dict[str, str | None] = {}
    seen_cursors: set[str] = set()
    pending: deque[str | None] = deque([None])
    calls = 0
    conversation: str | None = None
    while pending and calls < request_cap:
        cursor = pending.popleft()
        calls += 1
        body = _request(extractor, focal_id, cursor, variables, toggles)
        if capture_dir is not None:
            _capture_page(capture_dir, calls, body)
        page = parse_page(body)
        if conversation is None:
            conversation = _focal_conversation(page, focal_id)
        for index, tweet_id in enumerate(page.post_ids):
            if page.conversation_ids[index] != conversation:
                continue
            post_ids.add(tweet_id)
            reply_counts[tweet_id] = page.reply_counts[index]
            reply_to[tweet_id] = page.reply_to_ids[index]
        for value in page.cursors:
            if value in seen_cursors:
                continue
            seen_cursors.add(value)
            pending.append(value)
    reason = "request-cap" if pending else "exhausted"
    return post_ids, reply_counts, reply_to, calls, reason


def _capture_page(capture_dir: Path, ordinal: int, body: dict[str, Any]) -> None:
    """Write one response page so a run can be inspected afterwards.

    The page holds tweet text and cursor values. It stays in scratch and
    is never written to the vault.
    """
    capture_dir.mkdir(parents=True, exist_ok=True)
    target = capture_dir / f"page-{ordinal:03d}.json"
    target.write_text(
        json.dumps(body, indent=1, ensure_ascii=False) + "\n",
        encoding="utf-8",
    )


def _focal_conversation(page: _Page, focal_id: str) -> str:
    """Return the conversation id the focal's replies share.

    The focal's own conversation id is the one its replies carry. When
    the focal is not in the first page, the page's modal conversation
    id is used.
    """
    pairs = list(zip(page.post_ids, page.conversation_ids))
    for tweet_id, conversation in pairs:
        if tweet_id == focal_id and conversation:
            return conversation
    counts: dict[str, int] = {}
    for _tweet_id, conversation in pairs:
        if conversation:
            counts[conversation] = counts.get(conversation, 0) + 1
    if not counts:
        return ""
    return max(counts.items(), key=lambda item: item[1])[0]


def _under_captured(
    reply_counts: dict[str, int],
    reply_to: dict[str, str | None],
) -> list[str]:
    """Return posts whose reply count exceeds the children actually seen.

    A post that reports more replies than the walk found is a post with
    a subtree the walk has not reached yet. Walking it is how the walk
    expands without the operator naming every focal by hand.
    """
    children: dict[str, int] = {}
    for parent in reply_to.values():
        if parent:
            children[parent] = children.get(parent, 0) + 1
    return [
        post_id
        for post_id, count in reply_counts.items()
        if count > children.get(post_id, 0)
    ]

def discover(
    focal_ids: tuple[str, ...] | list[str],
    request_cap: int = 20,
    extractor: Any | None = None,
    max_focals: int = 40,
    capture_dir: Path | None = None,
) -> WalkerResult:
    """Walk each focal's conversation and return every post id found.

    The walk expands its own focal set: a post that reports more
    replies than were found has a subtree not yet reached, so it is
    walked too. That is what lets one focal find the whole thread.

    ``request_cap`` bounds the requests issued per focal.
    ``max_focals`` bounds how many focals the expansion may add, so a
    conversation that keeps reporting unreached children cannot run
    away. Requests are sequential. No cursor value is logged or
    persisted.
    """
    if not focal_ids:
        return WalkerResult(post_ids=(), calls=0, per_focal={}, stopped_reason="no-focals")
    variables, toggles = request_shape()
    if extractor is None:
        extractor = build_extractor()
    all_posts: set[str] = set()
    per_focal: dict[str, tuple[str, ...]] = {}
    calls = 0
    stopped = "exhausted"
    walked: set[str] = set()
    queue = list(dict.fromkeys(focal_ids))
    while queue and len(walked) < max_focals:
        focal = queue.pop(0)
        if focal in walked:
            continue
        walked.add(focal)
        posts, reply_counts, reply_to, focal_calls, reason = _walk_focal(
            extractor, focal, request_cap, variables, toggles, capture_dir
        )
        per_focal[focal] = tuple(sorted(posts))
        all_posts.update(posts)
        calls += focal_calls
        stopped = reason
        if len(walked) >= max_focals:
            stopped = "focal-cap"
            break
        for parent in _under_captured(reply_counts, reply_to):
            if parent not in walked:
                queue.append(parent)
    logger.info(
        "walker found %d posts across %d focals in %d requests",
        len(all_posts), len(walked), calls,
    )
    return WalkerResult(
        post_ids=tuple(sorted(all_posts)),
        calls=calls,
        per_focal=per_focal,
        stopped_reason=stopped,
    )