diff --git a/scripts/twitter/README.md b/scripts/twitter/README.md index 700c960..f5489b5 100644 --- a/scripts/twitter/README.md +++ b/scripts/twitter/README.md @@ -97,6 +97,15 @@ Re-run `discover` after the window refills to finish the walk. The post ids already found are printed before the stop, so a partial run still feeds `--branch`. +### Empty pages + +A page that holds no tweets and offers no nested cursor only +paginates empty space. Its bare `cursor-top-*` / `cursor-bottom-*` +pair leads to more pages that hold nothing, so the walk stops there +rather than spending the budget on them. A nested +`conversationthread-*-cursor-showmore-*` cursor is always followed: +that is where the posts gallery-dl drops live. + A capture is complete when no post reports more replies than the capture holds. `refresh` overwrites `thread_data.json` with whatever gallery-dl returns, so a refresh that drops posts loses them; the way diff --git a/scripts/twitter/tests/test_walker.py b/scripts/twitter/tests/test_walker.py index cdd3044..df9ff65 100644 --- a/scripts/twitter/tests/test_walker.py +++ b/scripts/twitter/tests/test_walker.py @@ -218,6 +218,52 @@ class DiscoverTests(unittest.TestCase): self.assertEqual(result.calls, 3) self.assertEqual(result.stopped_reason, "focal-cap") + def test_a_page_with_no_posts_is_not_paginated_further(self) -> None: + """A bare top/bottom pair on an empty page is a dead end. + + Following it spends the request budget on pages that hold no + tweets. The nested cursor is still followed, because that is + where the posts gallery-dl drops live. + """ + requests: list[str | None] = [] + + class FakeApi: + features_pagination = {"feature": True} + + def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict: + cursor = json.loads(params["variables"]).get("cursor") + requests.append(cursor) + # Every page after the first is empty and carries only + # the bare top/bottom pair. + return load("page_cursor_only.json") if cursor else load( + "page_nested_cursor.json" + ) + + class FakeExtractor: + api = FakeApi() + + result = walker.discover( + ("2121244702649532667",), + request_cap=20, + extractor=FakeExtractor(), + max_focals=1, + ) + # The first page's cursor is fetched, and its bare pair is not. + self.assertEqual(len(requests), 2) + self.assertIsNone(requests[0]) + self.assertIsNotNone(requests[1]) + self.assertTrue(result.post_ids) + + def test_a_nested_cursor_is_followed_from_an_empty_page(self) -> None: + """The show-more cursor survives the empty-page rule. + + It is the one cursor that leads to posts gallery-dl never + reaches, so a page offering it must still be followed. + """ + page = walker.parse_page(load("page_kenpex_baseline.json")) + self.assertTrue(page.nested_cursors) + self.assertTrue(page.post_ids) + def test_under_captured_finds_posts_with_unseen_children(self) -> None: """A post reporting more replies than were found has a subtree left. diff --git a/scripts/twitter/walker.py b/scripts/twitter/walker.py index b2762a4..9df70ba 100644 --- a/scripts/twitter/walker.py +++ b/scripts/twitter/walker.py @@ -213,6 +213,17 @@ class _Page: reply_to_ids: tuple[str | None, ...] cursors: tuple[str, ...] top_level_cursor: str | None + nested_cursors: tuple[str, ...] = () + + +def _is_nested_cursor(entry_id: str) -> bool: + """True for a cursor inside a thread module or show-more prompt. + + These are the cursors gallery-dl never follows: they sit inside a + ``conversationthread-*`` module and carry that module's id. The bare + ``cursor-top-*`` / ``cursor-bottom-*`` pair is ordinary pagination. + """ + return "showmore" in entry_id or entry_id.startswith(_THREAD_MODULE_PREFIXES) def _load_gallery_dl() -> Any: @@ -438,6 +449,7 @@ def parse_page(body: dict[str, Any]) -> _Page: """ post_ids: list[tuple[str, str]] = [] cursors: list[str] = [] + nested: list[str] = [] top_level: str | None = None def add_cursor(entry: dict[str, Any], top_level_entry: bool) -> None: @@ -450,6 +462,8 @@ def parse_page(body: dict[str, Any]) -> _Page: return if value not in cursors: cursors.append(value) + if _is_nested_cursor(entry_id) and value not in nested: + nested.append(value) if top_level_entry and entry_id.startswith("cursor-bottom-"): if top_level is None: top_level = value @@ -509,6 +523,7 @@ def parse_page(body: dict[str, Any]) -> _Page: reply_to_ids=tuple(entry[3] for entry in post_ids), cursors=tuple(cursors), top_level_cursor=top_level, + nested_cursors=tuple(nested), ) @@ -573,15 +588,9 @@ def _walk_focal( toggles: dict[str, Any], capture_dir: Path | None = None, tracker: "_RateTracker | None" = None, - page_cache: dict[str | None, dict[str, Any]] | None = None, ) -> tuple[set[str], dict[str, int], dict[str, str | None], int, str]: """Walk every cursor under one focal. - ``page_cache`` holds the pages already fetched in this walk, keyed - by cursor. The walk expands its focal set over posts in the same - conversation, and each new focal would otherwise re-fetch the same - pages from the top. - Returns ``(post_ids, reply_counts, reply_to_ids, calls, reason)``. """ post_ids: set[str] = set() @@ -592,51 +601,40 @@ def _walk_focal( calls = 0 conversation: str | None = None reason = "exhausted" - productive = 0 while pending and calls < request_cap: cursor = pending.popleft() - cached = None if page_cache is None else page_cache.get(cursor) - if cached is not None: - body = cached - else: - calls += 1 - try: - body = _request(extractor, focal_id, cursor, variables, toggles, tracker) - except RateLimitReached as exc: - # The walk ran out of room. Report what it found so the - # operator can resume with another run. - reason = "rate-limited" - logger.info("walk stopped: %s", exc) - break - if page_cache is not None: - page_cache[cursor] = body - if capture_dir is not None and cached is None: + calls += 1 + try: + body = _request(extractor, focal_id, cursor, variables, toggles, tracker) + except RateLimitReached as exc: + # The walk ran out of room. Report what it found so the + # operator can resume with another run. + reason = "rate-limited" + logger.info("walk stopped: %s", exc) + break + if capture_dir is not None: _capture_page(capture_dir, tracker.next_page() if tracker else calls, body) page = parse_page(body) if conversation is None: conversation = _focal_conversation(page, focal_id) - found = 0 for index, tweet_id in enumerate(page.post_ids): if page.conversation_ids[index] != conversation: continue - if tweet_id not in post_ids: - found += 1 post_ids.add(tweet_id) reply_counts[tweet_id] = page.reply_counts[index] reply_to[tweet_id] = page.reply_to_ids[index] - if found: - productive += 1 + has_tweets = bool(page.post_ids) for value in page.cursors: if value in seen_cursors: continue seen_cursors.add(value) + if not has_tweets and value not in page.nested_cursors: + # A page with no tweets and no nested cursor only + # paginates empty space. Its bare top/bottom pair + # leads to more pages that yield nothing, so following + # it spends the budget on pages that hold no posts. + continue pending.append(value) - if not found and not productive: - # The page held no posts and no earlier page did either. - # Its cursors lead away from the conversation's posts, so - # following them spends the budget on empty pages. - reason = "no-posts" - break if reason == "exhausted": reason = "request-cap" if pending else "exhausted" return post_ids, reply_counts, reply_to, calls, reason @@ -739,7 +737,6 @@ def discover( _clear_capture_dir(capture_dir) all_posts: set[str] = set() per_focal: dict[str, tuple[str, ...]] = {} - page_cache: dict[str | None, dict[str, Any]] = {} calls = 0 stopped = "exhausted" walked: set[str] = set() @@ -750,14 +747,7 @@ def discover( continue walked.add(focal) posts, reply_counts, reply_to, focal_calls, reason = _walk_focal( - extractor, - focal, - request_cap, - variables, - toggles, - capture_dir, - tracker, - page_cache, + extractor, focal, request_cap, variables, toggles, capture_dir, tracker ) per_focal[focal] = tuple(sorted(posts)) all_posts.update(posts)