walker adjustments
This commit is contained in:
1 parent
2343468bcd
commit
950d4e6466
3 files changed
+88
-43
No files matched your search
@@ -97,6 +97,15 @@ Re-run `discover` after the window refills to finish the walk. The
|
|||||||
post ids already found are printed before the stop, so a partial run
|
post ids already found are printed before the stop, so a partial run
|
||||||
still feeds `--branch`.
|
still feeds `--branch`.
|
||||||
|
|
||||||
|
### Empty pages
|
||||||
|
|
||||||
|
A page that holds no tweets and offers no nested cursor only
|
||||||
|
paginates empty space. Its bare `cursor-top-*` / `cursor-bottom-*`
|
||||||
|
pair leads to more pages that hold nothing, so the walk stops there
|
||||||
|
rather than spending the budget on them. A nested
|
||||||
|
`conversationthread-*-cursor-showmore-*` cursor is always followed:
|
||||||
|
that is where the posts gallery-dl drops live.
|
||||||
|
|
||||||
A capture is complete when no post reports more replies than the
|
A capture is complete when no post reports more replies than the
|
||||||
capture holds. `refresh` overwrites `thread_data.json` with whatever
|
capture holds. `refresh` overwrites `thread_data.json` with whatever
|
||||||
gallery-dl returns, so a refresh that drops posts loses them; the way
|
gallery-dl returns, so a refresh that drops posts loses them; the way
|
||||||
|
|||||||
@@ -218,6 +218,52 @@ class DiscoverTests(unittest.TestCase):
|
|||||||
self.assertEqual(result.calls, 3)
|
self.assertEqual(result.calls, 3)
|
||||||
self.assertEqual(result.stopped_reason, "focal-cap")
|
self.assertEqual(result.stopped_reason, "focal-cap")
|
||||||
|
|
||||||
|
def test_a_page_with_no_posts_is_not_paginated_further(self) -> None:
|
||||||
|
"""A bare top/bottom pair on an empty page is a dead end.
|
||||||
|
|
||||||
|
Following it spends the request budget on pages that hold no
|
||||||
|
tweets. The nested cursor is still followed, because that is
|
||||||
|
where the posts gallery-dl drops live.
|
||||||
|
"""
|
||||||
|
requests: list[str | None] = []
|
||||||
|
|
||||||
|
class FakeApi:
|
||||||
|
features_pagination = {"feature": True}
|
||||||
|
|
||||||
|
def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict:
|
||||||
|
cursor = json.loads(params["variables"]).get("cursor")
|
||||||
|
requests.append(cursor)
|
||||||
|
# Every page after the first is empty and carries only
|
||||||
|
# the bare top/bottom pair.
|
||||||
|
return load("page_cursor_only.json") if cursor else load(
|
||||||
|
"page_nested_cursor.json"
|
||||||
|
)
|
||||||
|
|
||||||
|
class FakeExtractor:
|
||||||
|
api = FakeApi()
|
||||||
|
|
||||||
|
result = walker.discover(
|
||||||
|
("2121244702649532667",),
|
||||||
|
request_cap=20,
|
||||||
|
extractor=FakeExtractor(),
|
||||||
|
max_focals=1,
|
||||||
|
)
|
||||||
|
# The first page's cursor is fetched, and its bare pair is not.
|
||||||
|
self.assertEqual(len(requests), 2)
|
||||||
|
self.assertIsNone(requests[0])
|
||||||
|
self.assertIsNotNone(requests[1])
|
||||||
|
self.assertTrue(result.post_ids)
|
||||||
|
|
||||||
|
def test_a_nested_cursor_is_followed_from_an_empty_page(self) -> None:
|
||||||
|
"""The show-more cursor survives the empty-page rule.
|
||||||
|
|
||||||
|
It is the one cursor that leads to posts gallery-dl never
|
||||||
|
reaches, so a page offering it must still be followed.
|
||||||
|
"""
|
||||||
|
page = walker.parse_page(load("page_kenpex_baseline.json"))
|
||||||
|
self.assertTrue(page.nested_cursors)
|
||||||
|
self.assertTrue(page.post_ids)
|
||||||
|
|
||||||
def test_under_captured_finds_posts_with_unseen_children(self) -> None:
|
def test_under_captured_finds_posts_with_unseen_children(self) -> None:
|
||||||
"""A post reporting more replies than were found has a subtree left.
|
"""A post reporting more replies than were found has a subtree left.
|
||||||
|
|
||||||
|
|||||||
+33
-43
@@ -213,6 +213,17 @@ class _Page:
|
|||||||
reply_to_ids: tuple[str | None, ...]
|
reply_to_ids: tuple[str | None, ...]
|
||||||
cursors: tuple[str, ...]
|
cursors: tuple[str, ...]
|
||||||
top_level_cursor: str | None
|
top_level_cursor: str | None
|
||||||
|
nested_cursors: tuple[str, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
def _is_nested_cursor(entry_id: str) -> bool:
|
||||||
|
"""True for a cursor inside a thread module or show-more prompt.
|
||||||
|
|
||||||
|
These are the cursors gallery-dl never follows: they sit inside a
|
||||||
|
``conversationthread-*`` module and carry that module's id. The bare
|
||||||
|
``cursor-top-*`` / ``cursor-bottom-*`` pair is ordinary pagination.
|
||||||
|
"""
|
||||||
|
return "showmore" in entry_id or entry_id.startswith(_THREAD_MODULE_PREFIXES)
|
||||||
|
|
||||||
|
|
||||||
def _load_gallery_dl() -> Any:
|
def _load_gallery_dl() -> Any:
|
||||||
@@ -438,6 +449,7 @@ def parse_page(body: dict[str, Any]) -> _Page:
|
|||||||
"""
|
"""
|
||||||
post_ids: list[tuple[str, str]] = []
|
post_ids: list[tuple[str, str]] = []
|
||||||
cursors: list[str] = []
|
cursors: list[str] = []
|
||||||
|
nested: list[str] = []
|
||||||
top_level: str | None = None
|
top_level: str | None = None
|
||||||
|
|
||||||
def add_cursor(entry: dict[str, Any], top_level_entry: bool) -> None:
|
def add_cursor(entry: dict[str, Any], top_level_entry: bool) -> None:
|
||||||
@@ -450,6 +462,8 @@ def parse_page(body: dict[str, Any]) -> _Page:
|
|||||||
return
|
return
|
||||||
if value not in cursors:
|
if value not in cursors:
|
||||||
cursors.append(value)
|
cursors.append(value)
|
||||||
|
if _is_nested_cursor(entry_id) and value not in nested:
|
||||||
|
nested.append(value)
|
||||||
if top_level_entry and entry_id.startswith("cursor-bottom-"):
|
if top_level_entry and entry_id.startswith("cursor-bottom-"):
|
||||||
if top_level is None:
|
if top_level is None:
|
||||||
top_level = value
|
top_level = value
|
||||||
@@ -509,6 +523,7 @@ def parse_page(body: dict[str, Any]) -> _Page:
|
|||||||
reply_to_ids=tuple(entry[3] for entry in post_ids),
|
reply_to_ids=tuple(entry[3] for entry in post_ids),
|
||||||
cursors=tuple(cursors),
|
cursors=tuple(cursors),
|
||||||
top_level_cursor=top_level,
|
top_level_cursor=top_level,
|
||||||
|
nested_cursors=tuple(nested),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -573,15 +588,9 @@ def _walk_focal(
|
|||||||
toggles: dict[str, Any],
|
toggles: dict[str, Any],
|
||||||
capture_dir: Path | None = None,
|
capture_dir: Path | None = None,
|
||||||
tracker: "_RateTracker | None" = None,
|
tracker: "_RateTracker | None" = None,
|
||||||
page_cache: dict[str | None, dict[str, Any]] | None = None,
|
|
||||||
) -> tuple[set[str], dict[str, int], dict[str, str | None], int, str]:
|
) -> tuple[set[str], dict[str, int], dict[str, str | None], int, str]:
|
||||||
"""Walk every cursor under one focal.
|
"""Walk every cursor under one focal.
|
||||||
|
|
||||||
``page_cache`` holds the pages already fetched in this walk, keyed
|
|
||||||
by cursor. The walk expands its focal set over posts in the same
|
|
||||||
conversation, and each new focal would otherwise re-fetch the same
|
|
||||||
pages from the top.
|
|
||||||
|
|
||||||
Returns ``(post_ids, reply_counts, reply_to_ids, calls, reason)``.
|
Returns ``(post_ids, reply_counts, reply_to_ids, calls, reason)``.
|
||||||
"""
|
"""
|
||||||
post_ids: set[str] = set()
|
post_ids: set[str] = set()
|
||||||
@@ -592,51 +601,40 @@ def _walk_focal(
|
|||||||
calls = 0
|
calls = 0
|
||||||
conversation: str | None = None
|
conversation: str | None = None
|
||||||
reason = "exhausted"
|
reason = "exhausted"
|
||||||
productive = 0
|
|
||||||
while pending and calls < request_cap:
|
while pending and calls < request_cap:
|
||||||
cursor = pending.popleft()
|
cursor = pending.popleft()
|
||||||
cached = None if page_cache is None else page_cache.get(cursor)
|
calls += 1
|
||||||
if cached is not None:
|
try:
|
||||||
body = cached
|
body = _request(extractor, focal_id, cursor, variables, toggles, tracker)
|
||||||
else:
|
except RateLimitReached as exc:
|
||||||
calls += 1
|
# The walk ran out of room. Report what it found so the
|
||||||
try:
|
# operator can resume with another run.
|
||||||
body = _request(extractor, focal_id, cursor, variables, toggles, tracker)
|
reason = "rate-limited"
|
||||||
except RateLimitReached as exc:
|
logger.info("walk stopped: %s", exc)
|
||||||
# The walk ran out of room. Report what it found so the
|
break
|
||||||
# operator can resume with another run.
|
if capture_dir is not None:
|
||||||
reason = "rate-limited"
|
|
||||||
logger.info("walk stopped: %s", exc)
|
|
||||||
break
|
|
||||||
if page_cache is not None:
|
|
||||||
page_cache[cursor] = body
|
|
||||||
if capture_dir is not None and cached is None:
|
|
||||||
_capture_page(capture_dir, tracker.next_page() if tracker else calls, body)
|
_capture_page(capture_dir, tracker.next_page() if tracker else calls, body)
|
||||||
page = parse_page(body)
|
page = parse_page(body)
|
||||||
if conversation is None:
|
if conversation is None:
|
||||||
conversation = _focal_conversation(page, focal_id)
|
conversation = _focal_conversation(page, focal_id)
|
||||||
found = 0
|
|
||||||
for index, tweet_id in enumerate(page.post_ids):
|
for index, tweet_id in enumerate(page.post_ids):
|
||||||
if page.conversation_ids[index] != conversation:
|
if page.conversation_ids[index] != conversation:
|
||||||
continue
|
continue
|
||||||
if tweet_id not in post_ids:
|
|
||||||
found += 1
|
|
||||||
post_ids.add(tweet_id)
|
post_ids.add(tweet_id)
|
||||||
reply_counts[tweet_id] = page.reply_counts[index]
|
reply_counts[tweet_id] = page.reply_counts[index]
|
||||||
reply_to[tweet_id] = page.reply_to_ids[index]
|
reply_to[tweet_id] = page.reply_to_ids[index]
|
||||||
if found:
|
has_tweets = bool(page.post_ids)
|
||||||
productive += 1
|
|
||||||
for value in page.cursors:
|
for value in page.cursors:
|
||||||
if value in seen_cursors:
|
if value in seen_cursors:
|
||||||
continue
|
continue
|
||||||
seen_cursors.add(value)
|
seen_cursors.add(value)
|
||||||
|
if not has_tweets and value not in page.nested_cursors:
|
||||||
|
# A page with no tweets and no nested cursor only
|
||||||
|
# paginates empty space. Its bare top/bottom pair
|
||||||
|
# leads to more pages that yield nothing, so following
|
||||||
|
# it spends the budget on pages that hold no posts.
|
||||||
|
continue
|
||||||
pending.append(value)
|
pending.append(value)
|
||||||
if not found and not productive:
|
|
||||||
# The page held no posts and no earlier page did either.
|
|
||||||
# Its cursors lead away from the conversation's posts, so
|
|
||||||
# following them spends the budget on empty pages.
|
|
||||||
reason = "no-posts"
|
|
||||||
break
|
|
||||||
if reason == "exhausted":
|
if reason == "exhausted":
|
||||||
reason = "request-cap" if pending else "exhausted"
|
reason = "request-cap" if pending else "exhausted"
|
||||||
return post_ids, reply_counts, reply_to, calls, reason
|
return post_ids, reply_counts, reply_to, calls, reason
|
||||||
@@ -739,7 +737,6 @@ def discover(
|
|||||||
_clear_capture_dir(capture_dir)
|
_clear_capture_dir(capture_dir)
|
||||||
all_posts: set[str] = set()
|
all_posts: set[str] = set()
|
||||||
per_focal: dict[str, tuple[str, ...]] = {}
|
per_focal: dict[str, tuple[str, ...]] = {}
|
||||||
page_cache: dict[str | None, dict[str, Any]] = {}
|
|
||||||
calls = 0
|
calls = 0
|
||||||
stopped = "exhausted"
|
stopped = "exhausted"
|
||||||
walked: set[str] = set()
|
walked: set[str] = set()
|
||||||
@@ -750,14 +747,7 @@ def discover(
|
|||||||
continue
|
continue
|
||||||
walked.add(focal)
|
walked.add(focal)
|
||||||
posts, reply_counts, reply_to, focal_calls, reason = _walk_focal(
|
posts, reply_counts, reply_to, focal_calls, reason = _walk_focal(
|
||||||
extractor,
|
extractor, focal, request_cap, variables, toggles, capture_dir, tracker
|
||||||
focal,
|
|
||||||
request_cap,
|
|
||||||
variables,
|
|
||||||
toggles,
|
|
||||||
capture_dir,
|
|
||||||
tracker,
|
|
||||||
page_cache,
|
|
||||||
)
|
)
|
||||||
per_focal[focal] = tuple(sorted(posts))
|
per_focal[focal] = tuple(sorted(posts))
|
||||||
all_posts.update(posts)
|
all_posts.update(posts)
|
||||||
|
|||||||
Reference in new issue
Block a user