walker adjustments

This commit is contained in:
ed committed 2026-11-15 15:23:27 -05:00
1 parent 2343468bcd
commit 950d4e6466
3 files changed
+88 -43

No files matched your search

+9
View File
@@ -97,6 +97,15 @@ Re-run `discover` after the window refills to finish the walk. The
post ids already found are printed before the stop, so a partial run
still feeds `--branch`.
### Empty pages
A page that holds no tweets and offers no nested cursor only
paginates empty space. Its bare `cursor-top-*` / `cursor-bottom-*`
pair leads to more pages that hold nothing, so the walk stops there
rather than spending the budget on them. A nested
`conversationthread-*-cursor-showmore-*` cursor is always followed:
that is where the posts gallery-dl drops live.
A capture is complete when no post reports more replies than the
capture holds. `refresh` overwrites `thread_data.json` with whatever
gallery-dl returns, so a refresh that drops posts loses them; the way
+46
View File
@@ -218,6 +218,52 @@ class DiscoverTests(unittest.TestCase):
self.assertEqual(result.calls, 3)
self.assertEqual(result.stopped_reason, "focal-cap")
def test_a_page_with_no_posts_is_not_paginated_further(self) -> None:
"""A bare top/bottom pair on an empty page is a dead end.
Following it spends the request budget on pages that hold no
tweets. The nested cursor is still followed, because that is
where the posts gallery-dl drops live.
"""
requests: list[str | None] = []
class FakeApi:
features_pagination = {"feature": True}
def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict:
cursor = json.loads(params["variables"]).get("cursor")
requests.append(cursor)
# Every page after the first is empty and carries only
# the bare top/bottom pair.
return load("page_cursor_only.json") if cursor else load(
"page_nested_cursor.json"
)
class FakeExtractor:
api = FakeApi()
result = walker.discover(
("2121244702649532667",),
request_cap=20,
extractor=FakeExtractor(),
max_focals=1,
)
# The first page's cursor is fetched, and its bare pair is not.
self.assertEqual(len(requests), 2)
self.assertIsNone(requests[0])
self.assertIsNotNone(requests[1])
self.assertTrue(result.post_ids)
def test_a_nested_cursor_is_followed_from_an_empty_page(self) -> None:
"""The show-more cursor survives the empty-page rule.
It is the one cursor that leads to posts gallery-dl never
reaches, so a page offering it must still be followed.
"""
page = walker.parse_page(load("page_kenpex_baseline.json"))
self.assertTrue(page.nested_cursors)
self.assertTrue(page.post_ids)
def test_under_captured_finds_posts_with_unseen_children(self) -> None:
"""A post reporting more replies than were found has a subtree left.
+33 -43
View File
@@ -213,6 +213,17 @@ class _Page:
reply_to_ids: tuple[str | None, ...]
cursors: tuple[str, ...]
top_level_cursor: str | None
nested_cursors: tuple[str, ...] = ()
def _is_nested_cursor(entry_id: str) -> bool:
"""True for a cursor inside a thread module or show-more prompt.
These are the cursors gallery-dl never follows: they sit inside a
``conversationthread-*`` module and carry that module's id. The bare
``cursor-top-*`` / ``cursor-bottom-*`` pair is ordinary pagination.
"""
return "showmore" in entry_id or entry_id.startswith(_THREAD_MODULE_PREFIXES)
def _load_gallery_dl() -> Any:
@@ -438,6 +449,7 @@ def parse_page(body: dict[str, Any]) -> _Page:
"""
post_ids: list[tuple[str, str]] = []
cursors: list[str] = []
nested: list[str] = []
top_level: str | None = None
def add_cursor(entry: dict[str, Any], top_level_entry: bool) -> None:
@@ -450,6 +462,8 @@ def parse_page(body: dict[str, Any]) -> _Page:
return
if value not in cursors:
cursors.append(value)
if _is_nested_cursor(entry_id) and value not in nested:
nested.append(value)
if top_level_entry and entry_id.startswith("cursor-bottom-"):
if top_level is None:
top_level = value
@@ -509,6 +523,7 @@ def parse_page(body: dict[str, Any]) -> _Page:
reply_to_ids=tuple(entry[3] for entry in post_ids),
cursors=tuple(cursors),
top_level_cursor=top_level,
nested_cursors=tuple(nested),
)
@@ -573,15 +588,9 @@ def _walk_focal(
toggles: dict[str, Any],
capture_dir: Path | None = None,
tracker: "_RateTracker | None" = None,
page_cache: dict[str | None, dict[str, Any]] | None = None,
) -> tuple[set[str], dict[str, int], dict[str, str | None], int, str]:
"""Walk every cursor under one focal.
``page_cache`` holds the pages already fetched in this walk, keyed
by cursor. The walk expands its focal set over posts in the same
conversation, and each new focal would otherwise re-fetch the same
pages from the top.
Returns ``(post_ids, reply_counts, reply_to_ids, calls, reason)``.
"""
post_ids: set[str] = set()
@@ -592,51 +601,40 @@ def _walk_focal(
calls = 0
conversation: str | None = None
reason = "exhausted"
productive = 0
while pending and calls < request_cap:
cursor = pending.popleft()
cached = None if page_cache is None else page_cache.get(cursor)
if cached is not None:
body = cached
else:
calls += 1
try:
body = _request(extractor, focal_id, cursor, variables, toggles, tracker)
except RateLimitReached as exc:
# The walk ran out of room. Report what it found so the
# operator can resume with another run.
reason = "rate-limited"
logger.info("walk stopped: %s", exc)
break
if page_cache is not None:
page_cache[cursor] = body
if capture_dir is not None and cached is None:
calls += 1
try:
body = _request(extractor, focal_id, cursor, variables, toggles, tracker)
except RateLimitReached as exc:
# The walk ran out of room. Report what it found so the
# operator can resume with another run.
reason = "rate-limited"
logger.info("walk stopped: %s", exc)
break
if capture_dir is not None:
_capture_page(capture_dir, tracker.next_page() if tracker else calls, body)
page = parse_page(body)
if conversation is None:
conversation = _focal_conversation(page, focal_id)
found = 0
for index, tweet_id in enumerate(page.post_ids):
if page.conversation_ids[index] != conversation:
continue
if tweet_id not in post_ids:
found += 1
post_ids.add(tweet_id)
reply_counts[tweet_id] = page.reply_counts[index]
reply_to[tweet_id] = page.reply_to_ids[index]
if found:
productive += 1
has_tweets = bool(page.post_ids)
for value in page.cursors:
if value in seen_cursors:
continue
seen_cursors.add(value)
if not has_tweets and value not in page.nested_cursors:
# A page with no tweets and no nested cursor only
# paginates empty space. Its bare top/bottom pair
# leads to more pages that yield nothing, so following
# it spends the budget on pages that hold no posts.
continue
pending.append(value)
if not found and not productive:
# The page held no posts and no earlier page did either.
# Its cursors lead away from the conversation's posts, so
# following them spends the budget on empty pages.
reason = "no-posts"
break
if reason == "exhausted":
reason = "request-cap" if pending else "exhausted"
return post_ids, reply_counts, reply_to, calls, reason
@@ -739,7 +737,6 @@ def discover(
_clear_capture_dir(capture_dir)
all_posts: set[str] = set()
per_focal: dict[str, tuple[str, ...]] = {}
page_cache: dict[str | None, dict[str, Any]] = {}
calls = 0
stopped = "exhausted"
walked: set[str] = set()
@@ -750,14 +747,7 @@ def discover(
continue
walked.add(focal)
posts, reply_counts, reply_to, focal_calls, reason = _walk_focal(
extractor,
focal,
request_cap,
variables,
toggles,
capture_dir,
tracker,
page_cache,
extractor, focal, request_cap, variables, toggles, capture_dir, tracker
)
per_focal[focal] = tuple(sorted(posts))
all_posts.update(posts)