diff --git a/archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language/2026-10-13-Colonthreee-this-is-fascinate-is-this-a-bit-like-spir-v.md b/archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language/2026-10-13-Colonthreee-this-is-fascinate-is-this-a-bit-like-spir-v.md new file mode 100644 index 0000000..58669e8 --- /dev/null +++ b/archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language/2026-10-13-Colonthreee-this-is-fascinate-is-this-a-bit-like-spir-v.md @@ -0,0 +1,37 @@ +--- +title: "@NOTimothyLottes This is fascinate." +type: archive +source: twitter +source_url: "https://x.com/Colonthreee/status/2109900069486858664" +author: ":3" +handle: Colonthreee +post_id: "2109900069486858664" +date: 2026-10-13 +archived: 2026-11-14 +draft: false +tags: + - archive + - twitter + - NOTimothyLottes +description: "@NOTimothyLottes This is fascinate." +in_reply_to: "" +parent_post_id: "2109859799454330995" +--- + +## Source + +- URL: https://x.com/Colonthreee/status/2109900069486858664 +- Author: :3 (@Colonthreee) +- Posted: 2026-10-13 06:52:31 + +## Branch + +**1/** **@Colonthreee** ^2109900069486858664 + +**@NOTimothyLottes** + +This is fascinate. Is this a bit like SPIR-V/similar binaries? + +## Related + +- Spine: [[archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language]] diff --git a/archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language/2026-10-13-leonard_coder-do-you-have-some-source-code-exemple-to-share.md b/archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language/2026-10-13-leonard_coder-do-you-have-some-source-code-exemple-to-share.md new file mode 100644 index 0000000..72572f0 --- /dev/null +++ b/archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language/2026-10-13-leonard_coder-do-you-have-some-source-code-exemple-to-share.md @@ -0,0 +1,43 @@ +--- +title: "@NOTimothyLottes do you have some source code exemple to share, just to check the kind of \"human friendly\"? Isn't it just \"Timothy\" friendly? :)" +type: archive +source: twitter +source_url: "https://x.com/leonard_coder/status/2109913619697635371" +author: "Arnaud Carré" +handle: leonard_coder +post_id: "2109913619697635371" +date: 2026-10-13 +archived: 2026-11-14 +draft: false +tags: + - archive + - twitter + - NOTimothyLottes +description: "@NOTimothyLottes do you have some source code exemple to share, just to check the kind of \"human friendly\"? Isn't it just \"Timothy\" friendly? :)" +in_reply_to: "" +parent_post_id: "2109859799454330995" +--- + +## Source + +- URL: https://x.com/leonard_coder/status/2109913619697635371 +- Author: Arnaud Carré (@leonard_coder) +- Posted: 2026-10-13 07:46:21 + +## Branch + +**1/** **@leonard_coder** ^2109913619697635371 + +**@NOTimothyLottes** + +do you have some source code exemple to share, just to check the kind of "human friendly"? Isn't it just "Timothy" friendly? :) + +**2/** **@NOTimothyLottes** ^2110006918601465949 + +**@leonard_coder** + +When it converges enough that I'm not still changing stuff, I'll post some examples. + +## Related + +- Spine: [[archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language]] diff --git a/archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language/index.md b/archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language/index.md new file mode 100644 index 0000000..a1574a3 --- /dev/null +++ b/archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language/index.md @@ -0,0 +1,48 @@ +--- +title: "I'm finding that my zero effort compile language which is effectively precompiled 32-bit words of binary x86-64 code that get taped together interleaved with flexible 32-bit immediates or rip relative addresses, is converging to something 6502 like." +type: archive +source: twitter +source_url: "https://x.com/NOTimothyLottes/status/2109857584937062679" +author: "NOTimothyLottes" +handle: NOTimothyLottes +post_id: "2109857584937062679" +date: 2026-10-13 +archived: 2026-11-14 +draft: false +tags: + - archive + - twitter + - NOTimothyLottes +description: "I'm finding that my zero effort compile language which is effectively precompiled 32-bit words of binary x86-64 code that get taped together interleaved with flexible 32-bit immediates or rip relative addresses, is converging to something 6502 like." +in_reply_to: "" +--- + +## Source + +- URL: https://x.com/NOTimothyLottes/status/2109857584937062679 +- Author: NOTimothyLottes (@NOTimothyLottes) +- Posted: 2026-10-13 04:03:42 + +## Thread + +**1/** **@NOTimothyLottes** ^2109857584937062679 + +I'm finding that my zero effort compile language which is effectively precompiled 32-bit words of binary x86-64 code that get taped together interleaved with flexible 32-bit immediates or rip relative addresses, is converging to something 6502 like. + +**2/** **@NOTimothyLottes** ^2109858227764461593 + +In that I have an accumulator, and 2 other operands, not {X and Y}, but more like {operand, and data stack pointer}. Where a "dictionary" acts like named variable in the zero-page. + +**3/** **@NOTimothyLottes** ^2109858871657545805 + +At the point where a virtual ISA has a tiny opcode bitwidth, it becomes amenable to factoring out code generation. But it also maps a space (tiny opcodes) similar to the constraints the 6502 engineers played in. + +**4/** **@NOTimothyLottes** ^2109859256287531455 + +The domain becomes, instant compile, slightly slower, but still native like performance for a simple virtual ISA, and then ability to go easily directly to the physical machine for anything that needs raw assembly. + +**5/** **@NOTimothyLottes** ^2109859799454330995 + +Taken a few weeks of not working on the project to progress to a point where the syntax of the language is human friendly enough to be easily usable for simultaneous usage as a virtualized language and a powerful macro assembly (for CPU and GPU) at the same time. + +Branches: [[archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language/2026-10-13-Colonthreee-this-is-fascinate-is-this-a-bit-like-spir-v]], [[archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language/2026-10-13-leonard_coder-do-you-have-some-source-code-exemple-to-share]] diff --git a/archive/threads/NOTimothyLottes/index.md b/archive/threads/NOTimothyLottes/index.md index 1a241ea..146fd7b 100644 --- a/archive/threads/NOTimothyLottes/index.md +++ b/archive/threads/NOTimothyLottes/index.md @@ -157,5 +157,6 @@ tags: - [[archive/threads/NOTimothyLottes/2026-10-04-if-the-deck-1-did-4m-units-youd-only-need-a-2-5]] - [[archive/threads/NOTimothyLottes/2026-10-04-thread-serious-talk-on-steamdeck-2-speculation]] - [[archive/threads/NOTimothyLottes/2026-10-06-if-one-used-bc7-for-a-giant-vq-code-book-like-a]] +- [[archive/threads/NOTimothyLottes/2026-10-13-im-finding-that-my-zero-effort-compile-language]] - [[archive/threads/NOTimothyLottes/2026-11-07-not-a-joke-im-re-trying-development-through]] - [[archive/threads/NOTimothyLottes/2026-11-13-thankfully-a-friend-sent-this-to-me-otherwise-id]] diff --git a/scripts/twitter/README.md b/scripts/twitter/README.md index 05f4697..700c960 100644 --- a/scripts/twitter/README.md +++ b/scripts/twitter/README.md @@ -21,7 +21,7 @@ python scripts/twitter/tw.py reslug --all python scripts/twitter/tw.py reslug --all --apply python scripts/twitter/tw.py relabel --all python scripts/twitter/tw.py relabel --all --apply -python scripts/twitter/tw.py discover --id +python scripts/twitter/tw.py discover --id [--cap N] [--max-requests N] ``` `paths.py` finds the vault two parents above this file. Dumps live at `../manual_slop/docs/twitter`. Scratch lives at `../Threadwell-ai/scratch`. @@ -81,6 +81,22 @@ were found has a subtree not yet reached, so it is walked too. One `--id` reaches the whole thread. `discover` is read-only against the vault and prints no cookies and no cursor values. +### Rate limit + +The endpoint allows 300 requests per 15-minute window. The walk stays +inside that window: + +- requests are paced 5 seconds apart, the same pacing the gallery-dl + captures use; +- `--max-requests` (default 180) caps the whole walk, not one focal; +- a 429 raises instead of sleeping. gallery-dl's default is to wait + out the window and retry, which reads as a hung command; `discover` + stops, reports what it found, and prints the refill time. + +Re-run `discover` after the window refills to finish the walk. The +post ids already found are printed before the stop, so a partial run +still feeds `--branch`. + A capture is complete when no post reports more replies than the capture holds. `refresh` overwrites `thread_data.json` with whatever gallery-dl returns, so a refresh that drops posts loses them; the way diff --git a/scripts/twitter/tests/test_walker.py b/scripts/twitter/tests/test_walker.py index 9893881..cdd3044 100644 --- a/scripts/twitter/tests/test_walker.py +++ b/scripts/twitter/tests/test_walker.py @@ -41,6 +41,14 @@ def load(name: str) -> dict: return json.loads((FIXTURES / name).read_text(encoding="utf-8")) +class AbortExtraction(Exception): + """Stands in for gallery-dl's AbortExtraction. + + The walker matches the exception by class name, so the test needs a + class with that name. + """ + + class ParsePageTests(unittest.TestCase): """The page parser must find cursors at every depth.""" @@ -257,7 +265,7 @@ class DiscoverTests(unittest.TestCase): self.assertGreater(len(walked_focals), len(result.per_focal)) def test_capture_dir_records_the_pages(self) -> None: - """A run writes its response pages so it can be inspected after.""" + """A run writes one response page per request, none overwritten.""" import tempfile class FakeApi: @@ -271,17 +279,47 @@ class DiscoverTests(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp: target = Path(tmp) / "walker" - walker.discover( + result = walker.discover( ("2121244702649532667",), request_cap=2, extractor=FakeExtractor(), capture_dir=target, ) pages = sorted(target.glob("page-*.json")) - self.assertEqual(len(pages), 2) + self.assertEqual(len(pages), result.calls) body = json.loads(pages[0].read_text(encoding="utf-8")) self.assertIn("threaded_conversation_with_injections_v2", body["data"]) + def test_capture_dir_keeps_every_page_of_a_multi_focal_walk(self) -> None: + """A walk over several focals must not reuse a page file name. + + The page number counts the whole walk, not one focal, so the + evidence of one focal cannot overwrite another's. + """ + import tempfile + + class FakeApi: + features_pagination = {"feature": True} + + def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict: + return load("page_nested_cursor.json") + + class FakeExtractor: + api = FakeApi() + + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "walker" + result = walker.discover( + ("2121244702649532667",), + request_cap=2, + extractor=FakeExtractor(), + capture_dir=target, + ) + names = sorted(p.name for p in target.glob("page-*.json")) + self.assertEqual(len(names), len(set(names))) + self.assertEqual(len(names), result.calls) + self.assertGreater(len(result.per_focal), 1) + def test_posts_from_other_conversations_are_excluded(self) -> None: """A conversation page can pull in other conversations' posts. @@ -314,6 +352,91 @@ class DiscoverTests(unittest.TestCase): self.assertNotIn(post_id, other_posts) +class RateLimitTests(unittest.TestCase): + """The walk must stay inside the endpoint's rate-limit window.""" + + def test_budget_stops_the_walk(self) -> None: + """The whole walk is capped, not just one focal.""" + tracker = walker._RateTracker(max_requests=3) + tracker.check_budget() + tracker.check_budget() + tracker.check_budget() + with self.assertRaises(walker.RateLimitReached): + tracker.check_budget() + + def test_budget_error_names_the_budget(self) -> None: + tracker = walker._RateTracker(max_requests=1) + tracker.check_budget() + with self.assertRaises(walker.RateLimitReached) as caught: + tracker.check_budget() + self.assertIn("budget of 1", str(caught.exception)) + + def test_a_nearly_spent_window_stops_the_walk(self) -> None: + """The walk stops before it spends the last requests in a window.""" + tracker = walker._RateTracker(max_requests=100) + + class FakeResponse: + headers = { + "x-rate-limit-remaining": "5", + "x-rate-limit-reset": "1794705157", + } + + tracker.observe(FakeResponse()) + self.assertEqual(tracker.remaining, 5) + with self.assertRaises(walker.RateLimitReached) as caught: + tracker.check_budget() + self.assertIn("nearly spent", str(caught.exception)) + self.assertIn("5 left", str(caught.exception)) + + def test_the_refill_time_is_reported(self) -> None: + """The operator learns when the window refills.""" + tracker = walker._RateTracker(max_requests=100) + + class FakeResponse: + headers = { + "x-rate-limit-remaining": "0", + "x-rate-limit-reset": "1794705157", + } + + tracker.observe(FakeResponse()) + error = tracker.rate_limited_error() + self.assertIn("refills at", str(error)) + self.assertIsNotNone(error.reset_at) + + def test_gallery_dl_abort_becomes_a_rate_limit_error(self) -> None: + """gallery-dl's AbortExtraction is translated, not swallowed. + + With ``ratelimit=abort`` gallery-dl raises instead of sleeping + until the window refills. The walker turns that into a message + the operator can act on. + """ + tracker = walker._RateTracker(max_requests=100) + + class FakeApi: + features_pagination = {"feature": True} + + def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict: + raise AbortExtraction("Rate limit exceeded") + + class FakeExtractor: + api = FakeApi() + + with self.assertRaises(walker.RateLimitReached): + walker._request( + FakeExtractor(), + "2121244702649532667", + None, + walker.request_shape()[0], + walker.request_shape()[1], + tracker, + ) + + def test_the_walk_reports_the_refill_time(self) -> None: + tracker = walker._RateTracker(max_requests=100) + error = tracker.rate_limited_error() + self.assertIn("rate limit reached", str(error)) + + class KenpexRegressionTests(unittest.TestCase): """The kenpex conversation: the fixture NOTE.md section 5 asked for.""" diff --git a/scripts/twitter/tw.py b/scripts/twitter/tw.py index f202139..ecab98a 100644 --- a/scripts/twitter/tw.py +++ b/scripts/twitter/tw.py @@ -30,6 +30,7 @@ if str(_HERE) not in sys.path: from paths import COOKIES, DUMPS, FROZEN, HERE, SCRATCH, VAULT from models import PostData, ThreadData, load_thread +from walker import DEFAULT_MAX_REQUESTS try: from twitter.tree import spine_from_tip, spine_ids, spine_quote_ids @@ -773,6 +774,10 @@ def cmd_discover(args: argparse.Namespace) -> int: Read-only against the vault. Prints one post id per line to stdout so the output pipes straight into ``refresh --branch``. Captured response pages go to ``scratch/walker_/``. + + The walk stays inside the endpoint's rate-limit window: requests are + paced, and the walk stops with a message naming the refill time + rather than sleeping through it. """ focals = tuple(dict.fromkeys(args.ids)) if not focals: @@ -793,6 +798,7 @@ def cmd_discover(args: argparse.Namespace) -> int: focals, request_cap=args.cap, capture_dir=SCRATCH / f"walker_{focals[0]}", + max_requests=args.max_requests, ) except WalkerError as exc: raise SystemExit(f"walker failed: {exc}") from exc @@ -800,9 +806,15 @@ def cmd_discover(args: argparse.Namespace) -> int: print(post_id) print( f"posts={len(result.post_ids)} focals={len(focals)} " - f"requests={result.calls} stop={result.stopped_reason}", + f"requests={result.calls} http={result.http_requests} " + f"stop={result.stopped_reason}", file=sys.stderr, ) + if result.reset_at is not None: + refill = datetime.fromtimestamp( + result.reset_at, tz=timezone.utc + ).strftime("%Y-%m-%d %H:%M:%S UTC") + print(f"rate-limit window refills at {refill}", file=sys.stderr) return 0 @@ -1120,6 +1132,16 @@ def build_parser() -> argparse.ArgumentParser: default=20, help="maximum requests per focal (default 20)", ) + discover_p.add_argument( + "--max-requests", + type=int, + default=DEFAULT_MAX_REQUESTS, + help=( + "maximum requests for the whole walk (default " + f"{DEFAULT_MAX_REQUESTS}); keeps the walk inside one " + "rate-limit window" + ), + ) return parser diff --git a/scripts/twitter/walker.py b/scripts/twitter/walker.py index 89715f1..b2762a4 100644 --- a/scripts/twitter/walker.py +++ b/scripts/twitter/walker.py @@ -22,6 +22,7 @@ import json import logging import re import textwrap +import time from collections import deque from dataclasses import dataclass from pathlib import Path @@ -42,18 +43,165 @@ _gallery_dl: Any = None # variables template it reads from its own source. FOCAL_TWEET_ID = "__focal_tweet_id__" +#: X allows 300 TweetDetail requests per 15-minute window. Five seconds +#: between requests puts a walk at 180 per window, the same pacing the +#: gallery-dl CLI uses for twitter captures. +DEFAULT_REQUEST_INTERVAL = 5.0 + +#: Requests one walk may issue. The window holds 300 over 15 minutes, so +#: a walk capped here finishes inside one window without needing a +#: refill. +DEFAULT_MAX_REQUESTS = 180 + +#: Remaining-request count at which a walk stops and waits for the +#: rate-limit window to refill. gallery-dl treats "remaining < 6" the +#: same way. +RATE_RESERVE = 6 + class WalkerError(RuntimeError): """Raised when the walker cannot build or run a request.""" +class RateLimitReached(WalkerError): + """Raised when the walk must stop because the request budget ran out. + + ``reset_at`` is the epoch second the endpoint's rate-limit window + refills, when the endpoint said so. + """ + + def __init__(self, message: str, reset_at: float | None = None) -> None: + super().__init__(message) + self.reset_at = reset_at + + +class _RateTracker: + """Counts the walk's requests and reads the endpoint's rate-limit window. + + ``TwitterAPI._call`` runs inside ``while True:``. On a 429, or on a + response whose ``x-rate-limit-remaining`` has nearly run out, it + calls ``_handle_ratelimit`` — which sleeps until the window refills + and then retries. A walk that blocks there looks hung and silently + burns the operator's time. + + The walk therefore configures gallery-dl to ``ratelimit=abort`` (so + it raises instead of sleeping) and to ``sleep-request`` (so it paces + itself). This tracker counts the requests that actually go out and + reads the window from every response, so the walk can stop before it + spends the last requests in a window and report when the window + refills. + """ + + def __init__( + self, + *, + max_requests: int = DEFAULT_MAX_REQUESTS, + ) -> None: + self.max_requests = max_requests + self.calls = 0 + self.pages = 0 + self.remaining: int | None = None + self.reset_at: float | None = None + + def observe(self, response: Any) -> None: + """Record the rate-limit headers of one response.""" + headers = getattr(response, "headers", None) + if not headers: + return + remaining = headers.get("x-rate-limit-remaining") + if remaining is not None: + try: + self.remaining = int(remaining) + except (TypeError, ValueError): + pass + reset = headers.get("x-rate-limit-reset") + if reset is not None: + try: + self.reset_at = float(reset) + except (TypeError, ValueError): + pass + + def install(self, extractor: Any) -> None: + """Watch the extractor's HTTP responses. + + The session is wrapped rather than ``Extractor.request`` so that + every request is counted, including the retries gallery-dl makes + on a 5xx. + + A 429 is raised here rather than returned. gallery-dl's request + layer retries a 429 with a 60-second sleep, four times over, + before the API layer ever sees it — four minutes of apparent + hang. Raising at the session boundary skips that loop. + """ + session = getattr(extractor, "session", None) + request = getattr(session, "request", None) + if request is None: + return + original = request + + def observed(*args: Any, **kwargs: Any) -> Any: + self.check_budget() + response = original(*args, **kwargs) + self.observe(response) + if getattr(response, "status_code", None) == 429: + raise self.rate_limited_error() + return response + + session.request = observed + + def check_budget(self) -> None: + """Stop the walk before it spends past its budget or its window.""" + if self.calls >= self.max_requests: + raise RateLimitReached( + f"request budget of {self.max_requests} spent" + ) + if self.remaining is not None and self.remaining <= RATE_RESERVE: + raise RateLimitReached( + f"rate-limit window nearly spent ({self.remaining} left); " + f"it refills at {self.refill_time()}", + reset_at=self.reset_at, + ) + self.calls += 1 + + def refill_time(self) -> str: + """Return the refill time as a local timestamp, or a placeholder.""" + if self.reset_at is None: + return "an unknown time" + return time.strftime( + "%Y-%m-%d %H:%M:%S", time.localtime(self.reset_at) + ) + + def rate_limited_error(self) -> RateLimitReached: + """Return the error to raise when the endpoint refused the walk.""" + if self.remaining is not None: + detail = f"{self.remaining} requests left" + else: + detail = "the window is exhausted" + return RateLimitReached( + f"rate limit reached ({detail}); it refills at {self.refill_time()}", + reset_at=self.reset_at, + ) + + def next_page(self) -> int: + """Return a page number that no other page in this walk uses.""" + self.pages += 1 + return self.pages + + @dataclass(frozen=True) class WalkerResult: - """What one walk found.""" + """What one walk found. + + ``calls`` counts the API requests the walk made. ``http_requests`` + counts the HTTP requests that reached the endpoint, which is the + number the rate-limit window counts. + """ post_ids: tuple[str, ...] calls: int per_focal: dict[str, tuple[str, ...]] stopped_reason: str + reset_at: float | None = None + http_requests: int = 0 @dataclass(frozen=True) @@ -155,14 +303,30 @@ def request_shape() -> tuple[dict[str, Any], dict[str, Any]]: return variables, toggles -def build_extractor() -> Any: +def build_extractor( + *, + request_interval: float = DEFAULT_REQUEST_INTERVAL, +) -> Any: """Build a gallery-dl extractor that is authenticated with the vault cookies. Cookies, the CSRF token, and the transaction id all come from gallery-dl's own request layer, so the walker speaks the same protocol gallery-dl does. + + Two settings keep the walk inside the endpoint's rate-limit window: + + - ``sleep-request`` paces the requests. The endpoint allows a fixed + number per window, so an unpaced walk exhausts it. + - ``ratelimit`` is set to ``abort``. Left at its default, a 429 makes + gallery-dl sleep until the window refills and then retry, inside a + ``while True:`` loop — the walk looks hung. Aborting lets the + walker report the refill time instead. """ twitter = _load_gallery_dl() + from gallery_dl import config as gd_config + + gd_config.set(("extractor", "twitter"), "sleep-request", request_interval) + gd_config.set(("extractor", "twitter"), "ratelimit", "abort") url = "https://x.com/i/web/status/1" extractor = twitter.TwitterTweetExtractor.from_url(url) if extractor is None: @@ -371,6 +535,7 @@ def _request( cursor: str | None, variables: dict[str, Any], toggles: dict[str, Any], + tracker: "_RateTracker | None" = None, ) -> dict[str, Any]: """Issue one TweetDetail request through gallery-dl's request layer.""" request_variables = dict(variables) @@ -384,12 +549,22 @@ def _request( ), "fieldToggles": json.dumps(toggles, separators=(",", ":"), ensure_ascii=False), } - body = extractor.api._call(tweet_detail_endpoint(), params, method="GET") + try: + body = extractor.api._call(tweet_detail_endpoint(), params, method="GET") + except Exception as exc: + if tracker is not None and _is_rate_limit_abort(exc): + raise tracker.rate_limited_error() from exc + raise if not isinstance(body, dict): raise WalkerError("TweetDetail returned a non-object body") return body +def _is_rate_limit_abort(exc: Exception) -> bool: + """True when gallery-dl aborted because the rate limit was hit.""" + return type(exc).__name__ == "AbortExtraction" + + def _walk_focal( extractor: Any, focal_id: str, @@ -397,9 +572,16 @@ def _walk_focal( variables: dict[str, Any], toggles: dict[str, Any], capture_dir: Path | None = None, + tracker: "_RateTracker | None" = None, + page_cache: dict[str | None, dict[str, Any]] | None = None, ) -> tuple[set[str], dict[str, int], dict[str, str | None], int, str]: """Walk every cursor under one focal. + ``page_cache`` holds the pages already fetched in this walk, keyed + by cursor. The walk expands its focal set over posts in the same + conversation, and each new focal would otherwise re-fetch the same + pages from the top. + Returns ``(post_ids, reply_counts, reply_to_ids, calls, reason)``. """ post_ids: set[str] = set() @@ -409,30 +591,65 @@ def _walk_focal( pending: deque[str | None] = deque([None]) calls = 0 conversation: str | None = None + reason = "exhausted" + productive = 0 while pending and calls < request_cap: cursor = pending.popleft() - calls += 1 - body = _request(extractor, focal_id, cursor, variables, toggles) - if capture_dir is not None: - _capture_page(capture_dir, calls, body) + cached = None if page_cache is None else page_cache.get(cursor) + if cached is not None: + body = cached + else: + calls += 1 + try: + body = _request(extractor, focal_id, cursor, variables, toggles, tracker) + except RateLimitReached as exc: + # The walk ran out of room. Report what it found so the + # operator can resume with another run. + reason = "rate-limited" + logger.info("walk stopped: %s", exc) + break + if page_cache is not None: + page_cache[cursor] = body + if capture_dir is not None and cached is None: + _capture_page(capture_dir, tracker.next_page() if tracker else calls, body) page = parse_page(body) if conversation is None: conversation = _focal_conversation(page, focal_id) + found = 0 for index, tweet_id in enumerate(page.post_ids): if page.conversation_ids[index] != conversation: continue + if tweet_id not in post_ids: + found += 1 post_ids.add(tweet_id) reply_counts[tweet_id] = page.reply_counts[index] reply_to[tweet_id] = page.reply_to_ids[index] + if found: + productive += 1 for value in page.cursors: if value in seen_cursors: continue seen_cursors.add(value) pending.append(value) - reason = "request-cap" if pending else "exhausted" + if not found and not productive: + # The page held no posts and no earlier page did either. + # Its cursors lead away from the conversation's posts, so + # following them spends the budget on empty pages. + reason = "no-posts" + break + if reason == "exhausted": + reason = "request-cap" if pending else "exhausted" return post_ids, reply_counts, reply_to, calls, reason +def _clear_capture_dir(capture_dir: Path) -> None: + """Remove the pages of a previous walk from the capture directory.""" + if not capture_dir.is_dir(): + return + for path in capture_dir.glob("page-*.json"): + path.unlink() + + def _capture_page(capture_dir: Path, ordinal: int, body: dict[str, Any]) -> None: """Write one response page so a run can be inspected afterwards. @@ -493,6 +710,7 @@ def discover( extractor: Any | None = None, max_focals: int = 40, capture_dir: Path | None = None, + max_requests: int = DEFAULT_MAX_REQUESTS, ) -> WalkerResult: """Walk each focal's conversation and return every post id found. @@ -503,16 +721,25 @@ def discover( ``request_cap`` bounds the requests issued per focal. ``max_focals`` bounds how many focals the expansion may add, so a conversation that keeps reporting unreached children cannot run - away. Requests are sequential. No cursor value is logged or - persisted. + away. ``max_requests`` bounds the requests the whole walk may + issue, so it stays inside one rate-limit window. Requests are + sequential. No cursor value is logged or persisted. """ if not focal_ids: return WalkerResult(post_ids=(), calls=0, per_focal={}, stopped_reason="no-focals") variables, toggles = request_shape() if extractor is None: extractor = build_extractor() + tracker = _RateTracker(max_requests=max_requests) + tracker.install(extractor) + if capture_dir is not None: + # The capture dir holds one walk's evidence. A previous run's + # pages would otherwise sit beside this run's, with page + # numbers that no longer line up. + _clear_capture_dir(capture_dir) all_posts: set[str] = set() per_focal: dict[str, tuple[str, ...]] = {} + page_cache: dict[str | None, dict[str, Any]] = {} calls = 0 stopped = "exhausted" walked: set[str] = set() @@ -523,12 +750,21 @@ def discover( continue walked.add(focal) posts, reply_counts, reply_to, focal_calls, reason = _walk_focal( - extractor, focal, request_cap, variables, toggles, capture_dir + extractor, + focal, + request_cap, + variables, + toggles, + capture_dir, + tracker, + page_cache, ) per_focal[focal] = tuple(sorted(posts)) all_posts.update(posts) calls += focal_calls stopped = reason + if reason == "rate-limited": + break if len(walked) >= max_focals: stopped = "focal-cap" break @@ -544,4 +780,6 @@ def discover( calls=calls, per_focal=per_focal, stopped_reason=stopped, + reset_at=tracker.reset_at, + http_requests=tracker.calls, )