walker adjustments
This commit is contained in:
1 parent
2343468bcd
commit
950d4e6466
3 files changed
+88
-43
No files matched your search
@@ -97,6 +97,15 @@ Re-run `discover` after the window refills to finish the walk. The
|
||||
post ids already found are printed before the stop, so a partial run
|
||||
still feeds `--branch`.
|
||||
|
||||
### Empty pages
|
||||
|
||||
A page that holds no tweets and offers no nested cursor only
|
||||
paginates empty space. Its bare `cursor-top-*` / `cursor-bottom-*`
|
||||
pair leads to more pages that hold nothing, so the walk stops there
|
||||
rather than spending the budget on them. A nested
|
||||
`conversationthread-*-cursor-showmore-*` cursor is always followed:
|
||||
that is where the posts gallery-dl drops live.
|
||||
|
||||
A capture is complete when no post reports more replies than the
|
||||
capture holds. `refresh` overwrites `thread_data.json` with whatever
|
||||
gallery-dl returns, so a refresh that drops posts loses them; the way
|
||||
|
||||
@@ -218,6 +218,52 @@ class DiscoverTests(unittest.TestCase):
|
||||
self.assertEqual(result.calls, 3)
|
||||
self.assertEqual(result.stopped_reason, "focal-cap")
|
||||
|
||||
def test_a_page_with_no_posts_is_not_paginated_further(self) -> None:
|
||||
"""A bare top/bottom pair on an empty page is a dead end.
|
||||
|
||||
Following it spends the request budget on pages that hold no
|
||||
tweets. The nested cursor is still followed, because that is
|
||||
where the posts gallery-dl drops live.
|
||||
"""
|
||||
requests: list[str | None] = []
|
||||
|
||||
class FakeApi:
|
||||
features_pagination = {"feature": True}
|
||||
|
||||
def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict:
|
||||
cursor = json.loads(params["variables"]).get("cursor")
|
||||
requests.append(cursor)
|
||||
# Every page after the first is empty and carries only
|
||||
# the bare top/bottom pair.
|
||||
return load("page_cursor_only.json") if cursor else load(
|
||||
"page_nested_cursor.json"
|
||||
)
|
||||
|
||||
class FakeExtractor:
|
||||
api = FakeApi()
|
||||
|
||||
result = walker.discover(
|
||||
("2121244702649532667",),
|
||||
request_cap=20,
|
||||
extractor=FakeExtractor(),
|
||||
max_focals=1,
|
||||
)
|
||||
# The first page's cursor is fetched, and its bare pair is not.
|
||||
self.assertEqual(len(requests), 2)
|
||||
self.assertIsNone(requests[0])
|
||||
self.assertIsNotNone(requests[1])
|
||||
self.assertTrue(result.post_ids)
|
||||
|
||||
def test_a_nested_cursor_is_followed_from_an_empty_page(self) -> None:
|
||||
"""The show-more cursor survives the empty-page rule.
|
||||
|
||||
It is the one cursor that leads to posts gallery-dl never
|
||||
reaches, so a page offering it must still be followed.
|
||||
"""
|
||||
page = walker.parse_page(load("page_kenpex_baseline.json"))
|
||||
self.assertTrue(page.nested_cursors)
|
||||
self.assertTrue(page.post_ids)
|
||||
|
||||
def test_under_captured_finds_posts_with_unseen_children(self) -> None:
|
||||
"""A post reporting more replies than were found has a subtree left.
|
||||
|
||||
|
||||
+33
-43
@@ -213,6 +213,17 @@ class _Page:
|
||||
reply_to_ids: tuple[str | None, ...]
|
||||
cursors: tuple[str, ...]
|
||||
top_level_cursor: str | None
|
||||
nested_cursors: tuple[str, ...] = ()
|
||||
|
||||
|
||||
def _is_nested_cursor(entry_id: str) -> bool:
|
||||
"""True for a cursor inside a thread module or show-more prompt.
|
||||
|
||||
These are the cursors gallery-dl never follows: they sit inside a
|
||||
``conversationthread-*`` module and carry that module's id. The bare
|
||||
``cursor-top-*`` / ``cursor-bottom-*`` pair is ordinary pagination.
|
||||
"""
|
||||
return "showmore" in entry_id or entry_id.startswith(_THREAD_MODULE_PREFIXES)
|
||||
|
||||
|
||||
def _load_gallery_dl() -> Any:
|
||||
@@ -438,6 +449,7 @@ def parse_page(body: dict[str, Any]) -> _Page:
|
||||
"""
|
||||
post_ids: list[tuple[str, str]] = []
|
||||
cursors: list[str] = []
|
||||
nested: list[str] = []
|
||||
top_level: str | None = None
|
||||
|
||||
def add_cursor(entry: dict[str, Any], top_level_entry: bool) -> None:
|
||||
@@ -450,6 +462,8 @@ def parse_page(body: dict[str, Any]) -> _Page:
|
||||
return
|
||||
if value not in cursors:
|
||||
cursors.append(value)
|
||||
if _is_nested_cursor(entry_id) and value not in nested:
|
||||
nested.append(value)
|
||||
if top_level_entry and entry_id.startswith("cursor-bottom-"):
|
||||
if top_level is None:
|
||||
top_level = value
|
||||
@@ -509,6 +523,7 @@ def parse_page(body: dict[str, Any]) -> _Page:
|
||||
reply_to_ids=tuple(entry[3] for entry in post_ids),
|
||||
cursors=tuple(cursors),
|
||||
top_level_cursor=top_level,
|
||||
nested_cursors=tuple(nested),
|
||||
)
|
||||
|
||||
|
||||
@@ -573,15 +588,9 @@ def _walk_focal(
|
||||
toggles: dict[str, Any],
|
||||
capture_dir: Path | None = None,
|
||||
tracker: "_RateTracker | None" = None,
|
||||
page_cache: dict[str | None, dict[str, Any]] | None = None,
|
||||
) -> tuple[set[str], dict[str, int], dict[str, str | None], int, str]:
|
||||
"""Walk every cursor under one focal.
|
||||
|
||||
``page_cache`` holds the pages already fetched in this walk, keyed
|
||||
by cursor. The walk expands its focal set over posts in the same
|
||||
conversation, and each new focal would otherwise re-fetch the same
|
||||
pages from the top.
|
||||
|
||||
Returns ``(post_ids, reply_counts, reply_to_ids, calls, reason)``.
|
||||
"""
|
||||
post_ids: set[str] = set()
|
||||
@@ -592,51 +601,40 @@ def _walk_focal(
|
||||
calls = 0
|
||||
conversation: str | None = None
|
||||
reason = "exhausted"
|
||||
productive = 0
|
||||
while pending and calls < request_cap:
|
||||
cursor = pending.popleft()
|
||||
cached = None if page_cache is None else page_cache.get(cursor)
|
||||
if cached is not None:
|
||||
body = cached
|
||||
else:
|
||||
calls += 1
|
||||
try:
|
||||
body = _request(extractor, focal_id, cursor, variables, toggles, tracker)
|
||||
except RateLimitReached as exc:
|
||||
# The walk ran out of room. Report what it found so the
|
||||
# operator can resume with another run.
|
||||
reason = "rate-limited"
|
||||
logger.info("walk stopped: %s", exc)
|
||||
break
|
||||
if page_cache is not None:
|
||||
page_cache[cursor] = body
|
||||
if capture_dir is not None and cached is None:
|
||||
calls += 1
|
||||
try:
|
||||
body = _request(extractor, focal_id, cursor, variables, toggles, tracker)
|
||||
except RateLimitReached as exc:
|
||||
# The walk ran out of room. Report what it found so the
|
||||
# operator can resume with another run.
|
||||
reason = "rate-limited"
|
||||
logger.info("walk stopped: %s", exc)
|
||||
break
|
||||
if capture_dir is not None:
|
||||
_capture_page(capture_dir, tracker.next_page() if tracker else calls, body)
|
||||
page = parse_page(body)
|
||||
if conversation is None:
|
||||
conversation = _focal_conversation(page, focal_id)
|
||||
found = 0
|
||||
for index, tweet_id in enumerate(page.post_ids):
|
||||
if page.conversation_ids[index] != conversation:
|
||||
continue
|
||||
if tweet_id not in post_ids:
|
||||
found += 1
|
||||
post_ids.add(tweet_id)
|
||||
reply_counts[tweet_id] = page.reply_counts[index]
|
||||
reply_to[tweet_id] = page.reply_to_ids[index]
|
||||
if found:
|
||||
productive += 1
|
||||
has_tweets = bool(page.post_ids)
|
||||
for value in page.cursors:
|
||||
if value in seen_cursors:
|
||||
continue
|
||||
seen_cursors.add(value)
|
||||
if not has_tweets and value not in page.nested_cursors:
|
||||
# A page with no tweets and no nested cursor only
|
||||
# paginates empty space. Its bare top/bottom pair
|
||||
# leads to more pages that yield nothing, so following
|
||||
# it spends the budget on pages that hold no posts.
|
||||
continue
|
||||
pending.append(value)
|
||||
if not found and not productive:
|
||||
# The page held no posts and no earlier page did either.
|
||||
# Its cursors lead away from the conversation's posts, so
|
||||
# following them spends the budget on empty pages.
|
||||
reason = "no-posts"
|
||||
break
|
||||
if reason == "exhausted":
|
||||
reason = "request-cap" if pending else "exhausted"
|
||||
return post_ids, reply_counts, reply_to, calls, reason
|
||||
@@ -739,7 +737,6 @@ def discover(
|
||||
_clear_capture_dir(capture_dir)
|
||||
all_posts: set[str] = set()
|
||||
per_focal: dict[str, tuple[str, ...]] = {}
|
||||
page_cache: dict[str | None, dict[str, Any]] = {}
|
||||
calls = 0
|
||||
stopped = "exhausted"
|
||||
walked: set[str] = set()
|
||||
@@ -750,14 +747,7 @@ def discover(
|
||||
continue
|
||||
walked.add(focal)
|
||||
posts, reply_counts, reply_to, focal_calls, reason = _walk_focal(
|
||||
extractor,
|
||||
focal,
|
||||
request_cap,
|
||||
variables,
|
||||
toggles,
|
||||
capture_dir,
|
||||
tracker,
|
||||
page_cache,
|
||||
extractor, focal, request_cap, variables, toggles, capture_dir, tracker
|
||||
)
|
||||
per_focal[focal] = tuple(sorted(posts))
|
||||
all_posts.update(posts)
|
||||
|
||||
Reference in new issue
Block a user