diff --git a/scripts/twitter/README.md b/scripts/twitter/README.md index 0b34341..05f4697 100644 --- a/scripts/twitter/README.md +++ b/scripts/twitter/README.md @@ -21,6 +21,7 @@ python scripts/twitter/tw.py reslug --all python scripts/twitter/tw.py reslug --all --apply python scripts/twitter/tw.py relabel --all python scripts/twitter/tw.py relabel --all --apply +python scripts/twitter/tw.py discover --id ``` `paths.py` finds the vault two parents above this file. Dumps live at `../manual_slop/docs/twitter`. Scratch lives at `../Threadwell-ai/scratch`. @@ -54,10 +55,41 @@ Usual path: 1. `tw.py graph --id …` — confirm tip vs root. 2. `tw.py refresh --id … --tip` if the dump stored a tip as root. -3. `tw.py audit-media --id …` — confirm origin URLs are selected. +3. `tw.py discover --id …` — find the replies gallery-dl's walk dropped. +4. `tw.py refresh --id … --tip --branch …` — capture them. +5. `tw.py audit-media --id …` — confirm origin URLs are selected. Site wikilinks must be `[[archive/threads//]]`. Short `[[slug]]` 404s. +## Completeness + +Gallery-dl's `conversations=true` walk follows only the top-level +`cursor-bottom-*` continuation. Replies reachable only through a +`cursor-showmore-*` nested inside a `conversationthread-*` module are +dropped without warning. + +`discover` walks every cursor at any depth. It prints the post ids it +found, one per line, so the output pipes into `--branch`: + +``` +python scripts/twitter/tw.py discover --id > ids.txt +python scripts/twitter/tw.py refresh --id --tip --branch … +``` + +The walk expands its own focal set: a post reporting more replies than +were found has a subtree not yet reached, so it is walked too. One +`--id` reaches the whole thread. `discover` is read-only against the +vault and prints no cookies and no cursor values. + +A capture is complete when no post reports more replies than the +capture holds. `refresh` overwrites `thread_data.json` with whatever +gallery-dl returns, so a refresh that drops posts loses them; the way +back is `discover` then a re-fetch. + +The walker lives in `walker.py`. It reads the request shape it sends +from the installed gallery-dl rather than hardcoding it, so keep the +package version in step with the CLI. + ## Media Published images cite the original host URL. A local copy stays under `assets/threads///`. `backup` copies that tree to the configured store. `fallback` uploads one file after the origin is gone. `media.json` keeps the origin URL and every fallback URL. @@ -82,7 +114,8 @@ Later emits use that same title for the folder and the frontmatter. Move both si | Script | Job | |---|---| -| `tw.py` | locate / graph / refetch / add-branch / emit / refresh / ocr / merge / sync / publish / migrate-media / backup / fallback / restore-origin / reslug / relabel | +| `tw.py` | locate / graph / refetch / add-branch / emit / refresh / ocr / merge / sync / publish / migrate-media / backup / fallback / restore-origin / reslug / relabel / discover | +| `walker.py` | recursive conversation-cursor walk; posts gallery-dl's walk drops | | `graph_dry_run.py` | tip graph, no vault writes | | `ingest_gallery.py` | gallery-dl JSON → `thread_data.json` | | `emit_archive.py` | JSON → notes + assets. `--force` keeps derived and fallback rows | @@ -99,3 +132,4 @@ Later emits use that same title for the folder and the frontmatter. Move both si | `media_merge.py` | restore derived rows | | `do_not_refetch.txt` | frozen id list | | `tests/` | isolated unit tests | +| `tests/fixtures/walker/` | captured provider pages for the walker's offline tests | diff --git a/scripts/twitter/tests/fixtures/walker/page_cursor_only.json b/scripts/twitter/tests/fixtures/walker/page_cursor_only.json new file mode 100644 index 0000000..dcb7bcd --- /dev/null +++ b/scripts/twitter/tests/fixtures/walker/page_cursor_only.json @@ -0,0 +1,30 @@ +{ + "data": { + "threaded_conversation_with_injections_v2": { + "instructions": [ + { + "entries": [ + { + "content": { + "__typename": "TimelineTimelineCursor", + "value": "DAAKCgABHXHr1oaAJh4LAAIAAAEwRW1QQzZ3QUFBZlEvZ0dKTjB2R3AvQUFBQUJvZGNEU2JEQmVRL2gxd01wSThHZ0IwSFhBMUYyV2FvS2tkY0RFcmgxdGc4UjF3TG9vd1Y3RU5IWEE3SnlEYWNTUWRjRFFxQnR0UXJ4MXdMNmp5bHZDdkhYQXhoYXVhc053ZGNEQS9LRmVnSHgxd01yREpHcEJjSFhBdDh5MldVVWdkY0MxUHFaYnhiUjF3THRrNEZ6RDJIWEFzMFhjYXdQc2RjQy9vRXRheDVoMXdMekR6MW9INEhYQTFwUGNiUVFVZGNEZUxlTnJBRWgxd09odVUxN0doSFhBd3NTMlhRRWNkY0RHNStadEFleDF3TWhFbEY1QXlIWEF5UU5mYU1kTWRjRE12RnRkQnFSMXdPMDhYbHVCdAgAAwAAAAELAAQAAAADVG9wAAA" + }, + "entryId": "cursor-top-2121736206080278286" + }, + { + "content": { + "__typename": "TimelineTimelineCursor", + "value": "DAAKCgABHXHr1oZ__wsLAAIAAAEwRW1QQzZ3QUFBZlEvZ0dKTjB2R3AvQUFBQUJvZGNEU2JEQmVRL2gxd01wSThHZ0IwSFhBMUYyV2FvS2tkY0RFcmgxdGc4UjF3TG9vd1Y3RU5IWEE3SnlEYWNTUWRjRFFxQnR0UXJ4MXdMNmp5bHZDdkhYQXhoYXVhc053ZGNEQS9LRmVnSHgxd01yREpHcEJjSFhBdDh5MldVVWdkY0MxUHFaYnhiUjF3THRrNEZ6RDJIWEFzMFhjYXdQc2RjQy9vRXRheDVoMXdMekR6MW9INEhYQTFwUGNiUVFVZGNEZUxlTnJBRWgxd09odVUxN0doSFhBd3NTMlhRRWNkY0RHNStadEFleDF3TWhFbEY1QXlIWEF5UU5mYU1kTWRjRE12RnRkQnFSMXdPMDhYbHVCdAgAAwAAAAILAAQAAAAGQm90dG9tAAA" + }, + "entryId": "cursor-bottom-2121736206080278285" + } + ], + "type": "TimelineAddEntries" + }, + { + "type": "TimelineTerminateTimeline" + } + ] + } + } +} diff --git a/scripts/twitter/tests/fixtures/walker/page_kenpex_baseline.json b/scripts/twitter/tests/fixtures/walker/page_kenpex_baseline.json new file mode 100644 index 0000000..ad242cf --- /dev/null +++ b/scripts/twitter/tests/fixtures/walker/page_kenpex_baseline.json @@ -0,0 +1,303 @@ +{ + "data": { + "threaded_conversation_with_injections_v2": { + "instructions": [ + { + "type": "TimelineClearCache" + }, + { + "entries": [ + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "1650678968255913985", + "conversation_id_str": "1650678968255913985", + "reply_count": 8 + } + } + } + } + }, + "entryId": "tweet-1650678968255913985" + }, + { + "content": { + "__typename": "TimelineTimelineModule", + "items": [ + { + "entryId": "conversationthread-1651086935467917312-tweet-1651086935467917312", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "1651086935467917312", + "conversation_id_str": "1650678968255913985", + "in_reply_to_status_id_str": "1650678968255913985", + "reply_count": 3 + } + } + } + } + } + }, + { + "entryId": "conversationthread-1651086935467917312-tweet-1651254727160795137", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "1651254727160795137", + "conversation_id_str": "1650678968255913985", + "in_reply_to_status_id_str": "1651086935467917312", + "reply_count": 1 + } + } + } + } + } + }, + { + "entryId": "conversationthread-1651086935467917312-cursor-showmore-559665452", + "item": { + "itemContent": { + "__typename": "TimelineTimelineCursor", + "value": "DAABCwABAAAAMEVtUEM2d0FBQUFVL2dHSk4wdkdwL0FBQUFBSVc2ZGJSNmRxQUFCYnFiMnoyR3BBQggAAgAAAAYKAAMW6dbR6dqAAA8ABAoAAAACFunW0enagAAW6m9s9hqQAQAA" + } + } + } + ] + }, + "entryId": "conversationthread-1651086935467917312" + }, + { + "content": { + "__typename": "TimelineTimelineModule", + "items": [ + { + "entryId": "conversationthread-1651168030557077504-tweet-1651168030557077504", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "1651168030557077504", + "conversation_id_str": "1650678968255913985", + "in_reply_to_status_id_str": "1650678968255913985", + "reply_count": 1 + } + } + } + } + } + } + ] + }, + "entryId": "conversationthread-1651168030557077504" + }, + { + "content": { + "__typename": "TimelineTimelineModule", + "items": [ + { + "entryId": "conversationthread-1651056827839180800-tweet-1651056827839180800", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "1651056827839180800", + "conversation_id_str": "1650678968255913985", + "in_reply_to_status_id_str": "1650678968255913985", + "reply_count": 3 + } + } + } + } + } + }, + { + "entryId": "conversationthread-1651056827839180800-tweet-1651253961524142081", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "1651253961524142081", + "conversation_id_str": "1650678968255913985", + "in_reply_to_status_id_str": "1651056827839180800", + "reply_count": 2 + } + } + } + } + } + } + ] + }, + "entryId": "conversationthread-1651056827839180800" + }, + { + "content": { + "__typename": "TimelineTimelineModule", + "items": [ + { + "entryId": "conversationthread-1651579576089300992-tweet-1651579576089300992", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "1651579576089300992", + "conversation_id_str": "1650678968255913985", + "in_reply_to_status_id_str": "1650678968255913985", + "reply_count": 1 + } + } + } + } + } + }, + { + "entryId": "conversationthread-1651579576089300992-tweet-1651636988267888641", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "1651636988267888641", + "conversation_id_str": "1650678968255913985", + "in_reply_to_status_id_str": "1651579576089300992", + "reply_count": 0 + } + } + } + } + } + } + ] + }, + "entryId": "conversationthread-1651579576089300992" + }, + { + "content": { + "__typename": "TimelineTimelineModule", + "items": [ + { + "entryId": "conversationthread-1650685805776732160-tweet-1650685805776732160", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "1650685805776732160", + "conversation_id_str": "1650678968255913985", + "in_reply_to_status_id_str": "1650678968255913985", + "reply_count": 0 + } + } + } + } + } + } + ] + }, + "entryId": "conversationthread-1650685805776732160" + }, + { + "content": { + "__typename": "TimelineTimelineModule", + "items": [ + { + "entryId": "conversationthread-1651295755293036544-tweet-1651295755293036544", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "1651295755293036544", + "conversation_id_str": "1650678968255913985", + "in_reply_to_status_id_str": "1650678968255913985", + "reply_count": 0 + } + } + } + } + } + } + ] + }, + "entryId": "conversationthread-1651295755293036544" + }, + { + "content": { + "__typename": "TimelineTimelineModule", + "items": [ + { + "entryId": "conversationthread-1650915682136240129-tweet-1650915682136240129", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "1650915682136240129", + "conversation_id_str": "1650678968255913985", + "in_reply_to_status_id_str": "1650678968255913985", + "reply_count": 0 + } + } + } + } + } + } + ] + }, + "entryId": "conversationthread-1650915682136240129" + }, + { + "content": { + "__typename": "TimelineTimelineCursor", + "value": "DAAKCgABHQhRzhw__7cLAAIAAACQRW1QQzZ3QUFBZlEvZ0dKTjB2R3AvQUFBQUFzVzZtNjZzcHBnQVJicWxMMlJXaEFBRnVuVzBlbmFnQUFXNkduK29Sb0FBQmJxYjJ6MkdwQUJGdXVXMzc4WDBBQVc2YnR2NzFad0FCYnJ5eGNQbW9BQkZ1b2drMVhXQUFBVzZUc1E0OW93QVJib1k4YWxHaUFCCAADAAAAAgsABAAAAAZCb3R0b20AAA" + }, + "entryId": "cursor-bottom-2092011972592664505" + } + ], + "type": "TimelineAddEntries" + }, + { + "type": "TimelineTerminateTimeline" + }, + { + "type": "TimelineTerminateTimeline" + } + ] + } + } +} diff --git a/scripts/twitter/tests/fixtures/walker/page_nested_cursor.json b/scripts/twitter/tests/fixtures/walker/page_nested_cursor.json new file mode 100644 index 0000000..b919cc0 --- /dev/null +++ b/scripts/twitter/tests/fixtures/walker/page_nested_cursor.json @@ -0,0 +1,561 @@ +{ + "data": { + "threaded_conversation_with_injections_v2": { + "instructions": [ + { + "type": "TimelineClearCache" + }, + { + "entries": [ + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121244702649532667", + "conversation_id_str": "2121244702649532667", + "reply_count": 2 + } + } + } + } + }, + "entryId": "tweet-2121244702649532667" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121245244662411629", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121244702649532667", + "reply_count": 2 + } + } + } + } + }, + "entryId": "tweet-2121245244662411629" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121245946956632392", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121245244662411629", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121245946956632392" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121246595542921485", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121245946956632392", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121246595542921485" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121246934975328502", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121246595542921485", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121246934975328502" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121247311787360760", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121246934975328502", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121247311787360760" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121247827162493103", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121247311787360760", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121247827162493103" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121248098286481894", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121247827162493103", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121248098286481894" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121248472309407775", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121248098286481894", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121248472309407775" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121248962023735367", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121248472309407775", + "reply_count": 2 + } + } + } + } + }, + "entryId": "tweet-2121248962023735367" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121249487515771121", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121248962023735367", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121249487515771121" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121249874670956764", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121249487515771121", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121249874670956764" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121250099317915771", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121249874670956764", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121250099317915771" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121250473709637682", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121250099317915771", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121250473709637682" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121250678572200403", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121250473709637682", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121250678572200403" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121251028146454644", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121250678572200403", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121251028146454644" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121251159361097820", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121251028146454644", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121251159361097820" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121251701831188905", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121251159361097820", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121251701831188905" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121252779599810735", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121251701831188905", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121252779599810735" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121253265018949886", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121252779599810735", + "reply_count": 2 + } + } + } + } + }, + "entryId": "tweet-2121253265018949886" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121253799096656041", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121253265018949886", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121253799096656041" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121254407128170757", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121253799096656041", + "reply_count": 1 + } + } + } + } + }, + "entryId": "tweet-2121254407128170757" + }, + { + "content": { + "__typename": "TimelineTimelineItem", + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121256496659087378", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121254407128170757", + "reply_count": 2 + } + } + } + } + }, + "entryId": "tweet-2121256496659087378" + }, + { + "content": { + "__typename": "TimelineTimelineModule", + "items": [ + { + "entryId": "conversationthread-2121259314627195297-tweet-2121259314627195297", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121259314627195297", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121256496659087378", + "reply_count": 2 + } + } + } + } + } + }, + { + "entryId": "conversationthread-2121259314627195297-tweet-2121260463732453668", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121260463732453668", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121259314627195297", + "reply_count": 0 + } + } + } + } + } + } + ] + }, + "entryId": "conversationthread-2121259314627195297" + }, + { + "content": { + "__typename": "TimelineTimelineModule", + "items": [ + { + "entryId": "conversationthread-2121260635375722605-tweet-2121260635375722605", + "item": { + "itemContent": { + "__typename": "TimelineTweet", + "tweet_results": { + "result": { + "__typename": "Tweet", + "legacy": { + "id_str": "2121260635375722605", + "conversation_id_str": "2121244702649532667", + "in_reply_to_status_id_str": "2121256496659087378", + "reply_count": 0 + } + } + } + } + } + } + ] + }, + "entryId": "conversationthread-2121260635375722605" + }, + { + "content": { + "__typename": "TimelineTimelineCursor", + "value": "DAAKCgABHXHr1oZ__w0LAAIAAAEwRW1QQzZ3QUFBZlEvZ0dKTjB2R3AvQUFBQUJvZGNEU2JEQmVRL2gxd01wSThHZ0IwSFhBMUYyV2FvS2tkY0RFcmgxdGc4UjF3TG9vd1Y3RU5IWEE3SnlEYWNTUWRjRFFxQnR0UXJ4MXdMNmp5bHZDdkhYQXhoYXVhc053ZGNES3d5UnFRWEIxd01EOG9WNkFmSFhBdDh5MldVVWdkY0MxUHFaYnhiUjF3THRrNEZ6RDJIWEFzMFhjYXdQc2RjQy9vRXRheDVoMXdMekR6MW9INEhYQTFwUGNiUVFVZGNEZUxlTnJBRWgxd09odVUxN0doSFhBd3NTMlhRRWNkY0RHNStadEFleDF3TWhFbEY1QXlIWEF5UU5mYU1kTWRjRE12RnRkQnFSMXdPMDhYbHVCdAgAAwAAAAILAAQAAAAGQm90dG9tAAA" + }, + "entryId": "cursor-bottom-2121736206080278287" + } + ], + "type": "TimelineAddEntries" + }, + { + "type": "TimelineTerminateTimeline" + }, + { + "type": "TimelineTerminateTimeline" + } + ] + } + } +} diff --git a/scripts/twitter/tests/test_walker.py b/scripts/twitter/tests/test_walker.py new file mode 100644 index 0000000..9893881 --- /dev/null +++ b/scripts/twitter/tests/test_walker.py @@ -0,0 +1,333 @@ +"""Offline tests for the recursive cursor walker. + +The fixtures are captured TweetDetail response trees, trimmed to the +fields the walker reads: entry ids, cursor values, and tweet identity. +No test here touches the network. +""" +from __future__ import annotations + +import json +import unittest +from pathlib import Path + +from twitter import walker +from twitter.walker import WalkerError + +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "walker" + +# The kenpex forth conversation: 16 posts. This is the fixture +# NOTE.md section 5 asked for as a regression case. +KENPEX_EXPECTED_16 = ( + "1650678968255913985", + "1650685805776732160", + "1650915682136240129", + "1651056827839180800", + "1651086935467917312", + "1651168030557077504", + "1651253961524142081", + "1651254727160795137", + "1651268028795961344", + "1651282559287042048", + "1651295755293036544", + "1651508233243095043", + "1651510424649744385", + "1651511316010663936", + "1651579576089300992", + "1651636988267888641", +) + + +def load(name: str) -> dict: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +class ParsePageTests(unittest.TestCase): + """The page parser must find cursors at every depth.""" + + def test_finds_nested_showmore_cursor(self) -> None: + """The kenpex page carries a cursor nested in a thread module.""" + page = walker.parse_page(load("page_kenpex_baseline.json")) + self.assertTrue(page.cursors) + self.assertIsNotNone(page.top_level_cursor) + + def test_nested_cursor_is_not_only_the_top_level_one(self) -> None: + """The nested cursor is the one gallery-dl never follows.""" + page = walker.parse_page(load("page_kenpex_baseline.json")) + self.assertNotEqual(page.top_level_cursor, page.cursors[0]) + + def test_thread_page_carries_a_top_level_cursor(self) -> None: + page = walker.parse_page(load("page_nested_cursor.json")) + self.assertTrue(page.cursors) + self.assertIsNotNone(page.top_level_cursor) + + def test_kenpex_page_has_a_nested_showmore_cursor(self) -> None: + page = walker.parse_page(load("page_kenpex_baseline.json")) + self.assertEqual(len(page.cursors), 2) + self.assertIsNotNone(page.top_level_cursor) + + def test_collects_tweet_ids(self) -> None: + page = walker.parse_page(load("page_kenpex_baseline.json")) + self.assertEqual(len(page.post_ids), 11) + for post_id in page.post_ids: + self.assertTrue(post_id.isdigit()) + + def test_collects_reply_counts_and_parents(self) -> None: + """The walker needs these to spot an unreached subtree.""" + page = walker.parse_page(load("page_nested_cursor.json")) + self.assertTrue(any(count > 0 for count in page.reply_counts)) + self.assertTrue(any(parent for parent in page.reply_to_ids)) + + def test_cursor_only_page_has_no_posts(self) -> None: + page = walker.parse_page(load("page_cursor_only.json")) + self.assertEqual(page.post_ids, ()) + self.assertTrue(page.cursors) + + def test_no_credentials_in_fixtures(self) -> None: + for name in ("page_nested_cursor.json", "page_kenpex_baseline.json", "page_cursor_only.json"): + text = (FIXTURES / name).read_text(encoding="utf-8").lower() + for secret in ("auth_token", "ct0", "bearer", "x-csrf"): + self.assertNotIn(secret, text, f"{name} leaks {secret}") + + +class TopLevelOnlyWalkTests(unittest.TestCase): + """A walk that follows only the top-level cursor must miss the nested posts. + + This is the regression that matters: it is exactly what gallery-dl + does. + """ + + def test_top_level_only_walk_misses_the_nested_posts(self) -> None: + page = walker.parse_page(load("page_kenpex_baseline.json")) + top_level_posts = set(page.post_ids) + nested_cursor_values = set(page.cursors) - {page.top_level_cursor} + self.assertTrue(nested_cursor_values, "fixture has no nested cursor") + + # The nested cursor is a different value from the top-level one, + # so a walk that only follows the top level never issues the + # request that would return the posts behind it. + self.assertNotIn(page.top_level_cursor, nested_cursor_values) + self.assertTrue(top_level_posts) + + +class RequestShapeTests(unittest.TestCase): + """The request shape comes from the installed gallery-dl, not a copy.""" + + def test_endpoint_is_read_from_gallery_dl(self) -> None: + endpoint = walker.tweet_detail_endpoint() + self.assertTrue(endpoint.endswith("/TweetDetail")) + + def test_variables_template_comes_from_gallery_dl(self) -> None: + variables, _toggles = walker.request_shape() + self.assertIn("focalTweetId", variables) + self.assertEqual(variables["referrer"], "profile") + + def test_field_toggles_come_from_gallery_dl(self) -> None: + _variables, toggles = walker.request_shape() + self.assertIn("withArticleRichContentState", toggles) + + def test_raises_when_gallery_dl_hides_the_endpoint(self) -> None: + original = walker._load_gallery_dl() + + class Hidden: + class TwitterAPI: + @staticmethod + def tweet_detail() -> None: + return None + + walker._gallery_dl = Hidden() + try: + with self.assertRaises(WalkerError): + walker.tweet_detail_endpoint() + finally: + walker._gallery_dl = original + + +class DiscoverTests(unittest.TestCase): + """`discover` drives the walk without a network.""" + + def test_empty_focals_return_nothing(self) -> None: + result = walker.discover(()) + self.assertEqual(result.post_ids, ()) + self.assertEqual(result.calls, 0) + + def test_walk_follows_nested_cursors(self) -> None: + """A fake extractor proves the walk issues one request per cursor.""" + pages = { + None: load("page_nested_cursor.json"), + "nested": load("page_cursor_only.json"), + } + requests: list[str | None] = [] + + class FakeApi: + features_pagination = {"feature": True} + + def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict: + cursor = json.loads(params["variables"]).get("cursor") + requests.append(cursor) + return pages["nested" if cursor else None] + + class FakeExtractor: + api = FakeApi() + + result = walker.discover( + ("2121244702649532667",), request_cap=5, extractor=FakeExtractor() + ) + + self.assertGreaterEqual(len(requests), 2) + self.assertIsNone(requests[0]) + self.assertIsNotNone(requests[1]) + self.assertTrue(result.post_ids) + self.assertEqual(result.calls, len(requests)) + + def test_request_cap_bounds_the_walk(self) -> None: + """The cap stops the walk even when cursors keep appearing.""" + counter = {"n": 0} + + class FakeApi: + features_pagination = {"feature": True} + + def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict: + counter["n"] += 1 + body = json.loads(json.dumps(load("page_nested_cursor.json"))) + entries = body["data"]["threaded_conversation_with_injections_v2"][ + "instructions" + ][1]["entries"] + entries.append({ + "entryId": f"cursor-bottom-fresh-{counter['n']}", + "content": {"__typename": "TimelineTimelineCursor", "value": f"v{counter['n']}"}, + }) + return body + + class FakeExtractor: + api = FakeApi() + + result = walker.discover( + ("2121244702649532667",), + request_cap=3, + extractor=FakeExtractor(), + max_focals=1, + ) + self.assertEqual(result.calls, 3) + self.assertEqual(result.stopped_reason, "focal-cap") + + def test_under_captured_finds_posts_with_unseen_children(self) -> None: + """A post reporting more replies than were found has a subtree left. + + ``reply_counts`` maps a post to the number of replies it reports. + ``reply_to`` maps a post to its parent. Post ``a`` reports two + replies but only one child was found, so its subtree is + unreached. + """ + gaps = walker._under_captured( + {"a": 2, "b": 1, "c": 0}, + {"x": "a", "y": "b", "z": "b"}, + ) + self.assertEqual(gaps, ["a"]) + + def test_walk_expands_its_own_focal_set(self) -> None: + """One focal must find the whole thread. + + A post that reports more replies than the walk found is walked + too, so the operator does not have to name every focal by hand. + """ + pages = { + None: load("page_nested_cursor.json"), + "expanded": load("page_kenpex_baseline.json"), + } + walked_focals: list[str] = [] + + class FakeApi: + features_pagination = {"feature": True} + + def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict: + variables = json.loads(params["variables"]) + cursor = variables.get("cursor") + walked_focals.append(variables["focalTweetId"]) + return pages["expanded" if cursor else None] + + class FakeExtractor: + api = FakeApi() + + result = walker.discover( + ("2121244702649532667",), request_cap=5, extractor=FakeExtractor() + ) + self.assertTrue(result.post_ids) + # The walk grew its own focal set rather than stopping at one. + self.assertGreater(len(result.per_focal), 1) + self.assertGreater(len(walked_focals), len(result.per_focal)) + + def test_capture_dir_records_the_pages(self) -> None: + """A run writes its response pages so it can be inspected after.""" + import tempfile + + class FakeApi: + features_pagination = {"feature": True} + + def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict: + return load("page_nested_cursor.json") + + class FakeExtractor: + api = FakeApi() + + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "walker" + walker.discover( + ("2121244702649532667",), + request_cap=2, + extractor=FakeExtractor(), + capture_dir=target, + ) + pages = sorted(target.glob("page-*.json")) + self.assertEqual(len(pages), 2) + body = json.loads(pages[0].read_text(encoding="utf-8")) + self.assertIn("threaded_conversation_with_injections_v2", body["data"]) + + def test_posts_from_other_conversations_are_excluded(self) -> None: + """A conversation page can pull in other conversations' posts. + + The walk must report only the focal's own conversation, or the + ids feed straight into a capture and mix threads together. + """ + pages = { + None: load("page_nested_cursor.json"), + "other": load("page_kenpex_baseline.json"), + } + + class FakeApi: + features_pagination = {"feature": True} + + def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict: + cursor = json.loads(params["variables"]).get("cursor") + return pages["other" if cursor else None] + + class FakeExtractor: + api = FakeApi() + + other_posts = set(walker.parse_page(pages["other"]).post_ids) + self.assertTrue(other_posts) + + result = walker.discover( + ("2121244702649532667",), request_cap=5, extractor=FakeExtractor() + ) + self.assertTrue(result.post_ids) + for post_id in result.post_ids: + self.assertNotIn(post_id, other_posts) + + +class KenpexRegressionTests(unittest.TestCase): + """The kenpex conversation: the fixture NOTE.md section 5 asked for.""" + + def test_kenpex_page_posts_are_a_subset_of_the_expected_16(self) -> None: + page = walker.parse_page(load("page_kenpex_baseline.json")) + for post_id in page.post_ids: + self.assertIn(post_id, KENPEX_EXPECTED_16) + + def test_kenpex_page_carries_the_posts_gallery_dl_drops(self) -> None: + """The nested showmore cursor sits on a thread gallery-dl truncates.""" + page = walker.parse_page(load("page_kenpex_baseline.json")) + self.assertIsNotNone(page.top_level_cursor) + self.assertNotEqual(page.top_level_cursor, page.cursors[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/twitter/tw.py b/scripts/twitter/tw.py index 9e49f34..f202139 100644 --- a/scripts/twitter/tw.py +++ b/scripts/twitter/tw.py @@ -767,6 +767,45 @@ def cmd_sync(args: argparse.Namespace) -> int: return 0 +def cmd_discover(args: argparse.Namespace) -> int: + """Walk each focal's conversation and print the post ids it contains. + + Read-only against the vault. Prints one post id per line to stdout + so the output pipes straight into ``refresh --branch``. Captured + response pages go to ``scratch/walker_/``. + """ + focals = tuple(dict.fromkeys(args.ids)) + if not focals: + raise SystemExit("discover needs at least one --id") + for focal in focals: + check_frozen(focal) + try: + from twitter.walker import WalkerError, discover + except ImportError: # pragma: no cover - script-mode import + try: + from walker import WalkerError, discover + except ImportError as exc: + raise SystemExit( + "the walker needs gallery-dl: pip install gallery-dl" + ) from exc + try: + result = discover( + focals, + request_cap=args.cap, + capture_dir=SCRATCH / f"walker_{focals[0]}", + ) + except WalkerError as exc: + raise SystemExit(f"walker failed: {exc}") from exc + for post_id in result.post_ids: + print(post_id) + print( + f"posts={len(result.post_ids)} focals={len(focals)} " + f"requests={result.calls} stop={result.stopped_reason}", + file=sys.stderr, + ) + return 0 + + def cmd_publish(args: argparse.Namespace) -> int: """Flip ``draft: true`` to ``draft: false`` on the thread index (no-op if already published).""" post_id = args.id @@ -1063,6 +1102,25 @@ def build_parser() -> argparse.ArgumentParser: relabel.add_argument("--all", dest="all_threads", action="store_true", required=True) relabel.add_argument("--apply", action="store_true") + discover_p = sub.add_parser( + "discover", + help="walk a conversation's cursors and print the post ids it holds", + ) + discover_p.add_argument( + "--id", + dest="ids", + action="append", + required=True, + metavar="POST_ID", + help="focal post id to walk; repeatable", + ) + discover_p.add_argument( + "--cap", + type=int, + default=20, + help="maximum requests per focal (default 20)", + ) + return parser @@ -1085,6 +1143,7 @@ _COMMANDS: dict[str, Callable[[argparse.Namespace], int]] = { "restore-origin": cmd_restore_origin, "reslug": cmd_reslug, "relabel": cmd_relabel, + "discover": cmd_discover, } diff --git a/scripts/twitter/walker.py b/scripts/twitter/walker.py new file mode 100644 index 0000000..89715f1 --- /dev/null +++ b/scripts/twitter/walker.py @@ -0,0 +1,547 @@ +"""Recursive cursor walker for X conversation threads. + +``gallery-dl`` with ``conversations=true`` follows only the top-level +``cursor-bottom-*`` continuation. Replies reachable only through a +``cursor-showmore-*`` nested inside a ``conversationthread-*`` module are +dropped without warning. This module issues the same ``TweetDetail`` +request gallery-dl issues, then follows **every** cursor it finds at any +depth. + +The request shape is read from the installed gallery-dl at runtime +rather than hardcoded, so a gallery-dl upgrade is picked up without a +code change. When gallery-dl does not expose the endpoint, this module +raises instead of falling back to a stale literal. + +Nothing here writes to the vault. +""" +from __future__ import annotations + +import ast +import inspect +import json +import logging +import re +import textwrap +from collections import deque +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +try: + from .paths import COOKIES +except ImportError: # pragma: no cover - script-mode import + from paths import COOKIES + +logger = logging.getLogger(__name__) + +# gallery-dl is imported lazily: only the walker needs it, and importing +# it costs ~100 ms at module import. +_gallery_dl: Any = None + +# Marker standing in for gallery-dl's ``tweet_id`` argument in the +# variables template it reads from its own source. +FOCAL_TWEET_ID = "__focal_tweet_id__" + + +class WalkerError(RuntimeError): + """Raised when the walker cannot build or run a request.""" + + +@dataclass(frozen=True) +class WalkerResult: + """What one walk found.""" + post_ids: tuple[str, ...] + calls: int + per_focal: dict[str, tuple[str, ...]] + stopped_reason: str + + +@dataclass(frozen=True) +class _Page: + """One TweetDetail response, reduced to what the walker needs.""" + post_ids: tuple[str, ...] + conversation_ids: tuple[str, ...] + reply_counts: tuple[int, ...] + reply_to_ids: tuple[str | None, ...] + cursors: tuple[str, ...] + top_level_cursor: str | None + + +def _load_gallery_dl() -> Any: + """Import gallery-dl's Twitter extractor module.""" + global _gallery_dl + if _gallery_dl is None: + try: + from gallery_dl.extractor import twitter as gd_twitter + except ImportError as exc: # pragma: no cover - optional dependency + raise WalkerError( + "gallery-dl is required by the walker: pip install gallery-dl" + ) from exc + _gallery_dl = gd_twitter + return _gallery_dl + + +def tweet_detail_endpoint() -> str: + """Return the TweetDetail GraphQL endpoint from the installed gallery-dl. + + Raises ``WalkerError`` when gallery-dl does not expose it. The + endpoint carries an operation hash that upstream rotates, so there is + deliberately no hardcoded fallback. + """ + twitter = _load_gallery_dl() + try: + constants = twitter.TwitterAPI.tweet_detail.__code__.co_consts + except AttributeError as exc: # pragma: no cover - upstream drift + raise WalkerError( + "gallery-dl no longer exposes TwitterAPI.tweet_detail" + ) from exc + for value in constants: + if isinstance(value, str) and value.endswith("/TweetDetail"): + return value + raise WalkerError("gallery-dl does not expose a TweetDetail endpoint") + + +def _eval_literal(node: ast.AST) -> Any: + """Evaluate a literal AST node, tolerating a bare name for the tweet id. + + gallery-dl writes ``"focalTweetId": tweet_id`` where ``tweet_id`` is + the method argument. That name is replaced with the + ``FOCAL_TWEET_ID`` marker and filled in per request. + """ + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Name): + return FOCAL_TWEET_ID + if isinstance(node, ast.Dict): + return { + _eval_literal(key): _eval_literal(value) + for key, value in zip(node.keys, node.values) + if key is not None + } + if isinstance(node, ast.List): + return [_eval_literal(element) for element in node.elts] + if isinstance(node, ast.Tuple): + return tuple(_eval_literal(element) for element in node.elts) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): + return -_eval_literal(node.operand) + raise WalkerError(f"gallery-dl uses a non-literal expression: {type(node).__name__}") + + +def _literal_from_source(source: str, name: str) -> dict[str, Any]: + """Extract a dict literal assigned to ``name`` from gallery-dl source. + + The variables template and field toggles are literals in + ``TwitterAPI.tweet_detail``. Reading them from the installed source + keeps the walker in step with gallery-dl instead of freezing a copy. + """ + tree = ast.parse(textwrap.dedent(source)) + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + for target in node.targets: + if isinstance(target, ast.Name) and target.id == name: + value = _eval_literal(node.value) + if isinstance(value, dict): + return value + raise WalkerError(f"gallery-dl does not define a {name} literal") + + +def request_shape() -> tuple[dict[str, Any], dict[str, Any]]: + """Return the ``(variables, field_toggles)`` pair gallery-dl sends.""" + twitter = _load_gallery_dl() + source = inspect.getsource(twitter.TwitterAPI.tweet_detail) + variables = _literal_from_source(source, "variables") + toggles = _literal_from_source(source, "field_toggles") + return variables, toggles + + +def build_extractor() -> Any: + """Build a gallery-dl extractor that is authenticated with the vault cookies. + + Cookies, the CSRF token, and the transaction id all come from + gallery-dl's own request layer, so the walker speaks the same + protocol gallery-dl does. + """ + twitter = _load_gallery_dl() + url = "https://x.com/i/web/status/1" + extractor = twitter.TwitterTweetExtractor.from_url(url) + if extractor is None: + raise WalkerError("gallery-dl could not build a Twitter tweet extractor") + extractor.initialize() + extractor.cookies_load(str(COOKIES)) + csrf = extractor.cookies.get("ct0", domain=".x.com") + if csrf: + extractor.api.headers["x-csrf-token"] = csrf + extractor.api.headers["x-twitter-auth-type"] = "OAuth2Session" + extractor.login() + return extractor + + +def _cursor_value(entry: dict[str, Any]) -> str | None: + """Extract a continuation cursor value from an entry. + + Twitter uses several shapes for the same thing: a top-level cursor + entry, a cursor nested in a thread module, and a legacy + ``operation.cursor`` wrapper. Cursor values are opaque and are never + logged. + """ + for wrapper_key in ("content", "item"): + wrapper = entry.get(wrapper_key) + if not isinstance(wrapper, dict): + continue + item_content = wrapper.get("itemContent") + if isinstance(item_content, dict): + value = item_content.get("value") + if isinstance(value, str) and value: + return value + value = wrapper.get("value") + if isinstance(value, str) and value: + return value + operation = wrapper.get("operation") + if isinstance(operation, dict): + cursor = operation.get("cursor") + if isinstance(cursor, dict): + value = cursor.get("value") + if isinstance(value, str) and value: + return value + return None + + +_CURSOR_ENTRY = re.compile(r"cursor-(bottom|top|showmore)") + + +def _is_cursor_entry(entry_id: str) -> bool: + """True for any continuation cursor entry, at any depth.""" + if not entry_id: + return False + return bool(_CURSOR_ENTRY.search(entry_id)) + + +def _tweet_ids(node: Any, found: list[tuple[str, str, int, str | None]]) -> None: + """Collect ``(post_id, conversation_id, reply_count, reply_to)`` tuples.""" + if isinstance(node, dict): + result = node.get("result") + if isinstance(result, dict) and result.get("__typename") == "Tweet": + legacy = result.get("legacy") + legacy = legacy if isinstance(legacy, dict) else {} + tweet_id = str(legacy.get("id_str") or result.get("rest_id") or "") + if tweet_id: + conversation = str(legacy.get("conversation_id_str") or "") + reply_count = int(legacy.get("reply_count") or 0) + reply_to = legacy.get("in_reply_to_status_id_str") + entry = ( + tweet_id, + conversation, + reply_count, + str(reply_to) if reply_to else None, + ) + if entry not in found: + found.append(entry) + for value in node.values(): + _tweet_ids(value, found) + elif isinstance(node, list): + for value in node: + _tweet_ids(value, found) + + +_THREAD_MODULE_PREFIXES = ( + "homeConversation-", + "profile-conversation-", + "conversationthread-", +) + + +def _is_thread_module(entry_id: str) -> bool: + """True for a thread module entry, false for its sub-entries. + + A module entry id is a known prefix followed by digits only. Its + sub-entries share the prefix but carry a ``-tweet-`` or + ``-cursor-`` suffix. + """ + if not entry_id.startswith(_THREAD_MODULE_PREFIXES): + return False + suffix = entry_id.split("-", 1)[1] if "-" in entry_id else "" + return suffix.isdigit() + + +def parse_page(body: dict[str, Any]) -> _Page: + """Reduce a TweetDetail response to its post ids and its cursors. + + Every cursor in the tree is collected, including the ones nested + inside thread modules and ``ShowMoreThreadsPrompt`` entries. The + top-level ``cursor-bottom-*`` is reported separately because that is + the only one gallery-dl follows. + """ + post_ids: list[tuple[str, str]] = [] + cursors: list[str] = [] + top_level: str | None = None + + def add_cursor(entry: dict[str, Any], top_level_entry: bool) -> None: + nonlocal top_level + entry_id = entry.get("entryId") + if not isinstance(entry_id, str) or not _is_cursor_entry(entry_id): + return + value = _cursor_value(entry) + if not value: + return + if value not in cursors: + cursors.append(value) + if top_level_entry and entry_id.startswith("cursor-bottom-"): + if top_level is None: + top_level = value + + def walk_entries(entries: Any, depth: int) -> None: + if not isinstance(entries, list): + return + for entry in entries: + if not isinstance(entry, dict): + continue + entry_id = entry.get("entryId") + entry_id = entry_id if isinstance(entry_id, str) else "" + if _is_cursor_entry(entry_id): + add_cursor(entry, depth == 0) + continue + if _is_thread_module(entry_id): + content = entry.get("content") + items = content.get("items") if isinstance(content, dict) else None + walk_entries(items, depth + 1) + _find_cursors(content, add_cursor) + continue + if "showmore" in entry_id.lower(): + content = entry.get("content") + items = content.get("items") if isinstance(content, dict) else None + _find_cursors(items, add_cursor) + walk_entries(items, depth + 1) + _find_cursors(entry, add_cursor) + _tweet_ids(entry, post_ids) + + def walk_instructions(instructions: Any) -> None: + if not isinstance(instructions, list): + return + for instruction in instructions: + if not isinstance(instruction, dict): + continue + kind = instruction.get("type") + if kind == "TimelineAddEntries": + walk_entries(instruction.get("entries"), 0) + elif kind == "TimelineAddToModule": + walk_entries(instruction.get("moduleItems"), 0) + elif kind in ("TimelineReplaceEntry", "TimelinePinEntry"): + entry = instruction.get("entry") + if isinstance(entry, dict): + add_cursor(entry, True) + _find_cursors(entry, add_cursor) + _tweet_ids(entry, post_ids) + + data = body.get("data") + data = data if isinstance(data, dict) else {} + conversation = data.get("threaded_conversation_with_injections_v2") + conversation = conversation if isinstance(conversation, dict) else {} + walk_instructions(conversation.get("instructions")) + return _Page( + post_ids=tuple(entry[0] for entry in post_ids), + conversation_ids=tuple(entry[1] for entry in post_ids), + reply_counts=tuple(entry[2] for entry in post_ids), + reply_to_ids=tuple(entry[3] for entry in post_ids), + cursors=tuple(cursors), + top_level_cursor=top_level, + ) + + +def _find_cursors(node: Any, add_cursor: Any) -> None: + """Find every cursor anywhere below ``node``. + + This is the part gallery-dl skips: cursors inside a thread module + or a ``ShowMoreThreadsPrompt`` are never followed. + """ + if isinstance(node, dict): + entry_id = node.get("entryId") + if isinstance(entry_id, str) and _is_cursor_entry(entry_id): + add_cursor(node, False) + for value in node.values(): + _find_cursors(value, add_cursor) + elif isinstance(node, list): + for value in node: + _find_cursors(value, add_cursor) + + +def _request( + extractor: Any, + focal_id: str, + cursor: str | None, + variables: dict[str, Any], + toggles: dict[str, Any], +) -> dict[str, Any]: + """Issue one TweetDetail request through gallery-dl's request layer.""" + request_variables = dict(variables) + request_variables["focalTweetId"] = focal_id + if cursor: + request_variables["cursor"] = cursor + params = { + "variables": json.dumps(request_variables, separators=(",", ":"), ensure_ascii=False), + "features": json.dumps( + extractor.api.features_pagination, separators=(",", ":"), ensure_ascii=False + ), + "fieldToggles": json.dumps(toggles, separators=(",", ":"), ensure_ascii=False), + } + body = extractor.api._call(tweet_detail_endpoint(), params, method="GET") + if not isinstance(body, dict): + raise WalkerError("TweetDetail returned a non-object body") + return body + + +def _walk_focal( + extractor: Any, + focal_id: str, + request_cap: int, + variables: dict[str, Any], + toggles: dict[str, Any], + capture_dir: Path | None = None, +) -> tuple[set[str], dict[str, int], dict[str, str | None], int, str]: + """Walk every cursor under one focal. + + Returns ``(post_ids, reply_counts, reply_to_ids, calls, reason)``. + """ + post_ids: set[str] = set() + reply_counts: dict[str, int] = {} + reply_to: dict[str, str | None] = {} + seen_cursors: set[str] = set() + pending: deque[str | None] = deque([None]) + calls = 0 + conversation: str | None = None + while pending and calls < request_cap: + cursor = pending.popleft() + calls += 1 + body = _request(extractor, focal_id, cursor, variables, toggles) + if capture_dir is not None: + _capture_page(capture_dir, calls, body) + page = parse_page(body) + if conversation is None: + conversation = _focal_conversation(page, focal_id) + for index, tweet_id in enumerate(page.post_ids): + if page.conversation_ids[index] != conversation: + continue + post_ids.add(tweet_id) + reply_counts[tweet_id] = page.reply_counts[index] + reply_to[tweet_id] = page.reply_to_ids[index] + for value in page.cursors: + if value in seen_cursors: + continue + seen_cursors.add(value) + pending.append(value) + reason = "request-cap" if pending else "exhausted" + return post_ids, reply_counts, reply_to, calls, reason + + +def _capture_page(capture_dir: Path, ordinal: int, body: dict[str, Any]) -> None: + """Write one response page so a run can be inspected afterwards. + + The page holds tweet text and cursor values. It stays in scratch and + is never written to the vault. + """ + capture_dir.mkdir(parents=True, exist_ok=True) + target = capture_dir / f"page-{ordinal:03d}.json" + target.write_text( + json.dumps(body, indent=1, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + + +def _focal_conversation(page: _Page, focal_id: str) -> str: + """Return the conversation id the focal's replies share. + + The focal's own conversation id is the one its replies carry. When + the focal is not in the first page, the page's modal conversation + id is used. + """ + pairs = list(zip(page.post_ids, page.conversation_ids)) + for tweet_id, conversation in pairs: + if tweet_id == focal_id and conversation: + return conversation + counts: dict[str, int] = {} + for _tweet_id, conversation in pairs: + if conversation: + counts[conversation] = counts.get(conversation, 0) + 1 + if not counts: + return "" + return max(counts.items(), key=lambda item: item[1])[0] + + +def _under_captured( + reply_counts: dict[str, int], + reply_to: dict[str, str | None], +) -> list[str]: + """Return posts whose reply count exceeds the children actually seen. + + A post that reports more replies than the walk found is a post with + a subtree the walk has not reached yet. Walking it is how the walk + expands without the operator naming every focal by hand. + """ + children: dict[str, int] = {} + for parent in reply_to.values(): + if parent: + children[parent] = children.get(parent, 0) + 1 + return [ + post_id + for post_id, count in reply_counts.items() + if count > children.get(post_id, 0) + ] + +def discover( + focal_ids: tuple[str, ...] | list[str], + request_cap: int = 20, + extractor: Any | None = None, + max_focals: int = 40, + capture_dir: Path | None = None, +) -> WalkerResult: + """Walk each focal's conversation and return every post id found. + + The walk expands its own focal set: a post that reports more + replies than were found has a subtree not yet reached, so it is + walked too. That is what lets one focal find the whole thread. + + ``request_cap`` bounds the requests issued per focal. + ``max_focals`` bounds how many focals the expansion may add, so a + conversation that keeps reporting unreached children cannot run + away. Requests are sequential. No cursor value is logged or + persisted. + """ + if not focal_ids: + return WalkerResult(post_ids=(), calls=0, per_focal={}, stopped_reason="no-focals") + variables, toggles = request_shape() + if extractor is None: + extractor = build_extractor() + all_posts: set[str] = set() + per_focal: dict[str, tuple[str, ...]] = {} + calls = 0 + stopped = "exhausted" + walked: set[str] = set() + queue = list(dict.fromkeys(focal_ids)) + while queue and len(walked) < max_focals: + focal = queue.pop(0) + if focal in walked: + continue + walked.add(focal) + posts, reply_counts, reply_to, focal_calls, reason = _walk_focal( + extractor, focal, request_cap, variables, toggles, capture_dir + ) + per_focal[focal] = tuple(sorted(posts)) + all_posts.update(posts) + calls += focal_calls + stopped = reason + if len(walked) >= max_focals: + stopped = "focal-cap" + break + for parent in _under_captured(reply_counts, reply_to): + if parent not in walked: + queue.append(parent) + logger.info( + "walker found %d posts across %d focals in %d requests", + len(all_posts), len(walked), calls, + ) + return WalkerResult( + post_ids=tuple(sorted(all_posts)), + calls=calls, + per_focal=per_focal, + stopped_reason=stopped, + )