ed/Threadwell
1
0 Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

prompted walker from scratch.

Browse Source
This commit is contained in:
ed committed 2026-11-14 19:48:11 -05:00
1 parent 65b7e566ed
commit 84312401bd
7 files changed
+1869 -2
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/twitter/README.md
+36 -2
View File
@@ -21,6 +21,7 @@ python scripts/twitter/tw.py reslug --all
python scripts/twitter/tw.py reslug --all --apply
python scripts/twitter/tw.py relabel --all
python scripts/twitter/tw.py relabel --all --apply
python scripts/twitter/tw.py discover --id <post-id>
```
`paths.py` finds the vault two parents above this file. Dumps live at `../manual_slop/docs/twitter`. Scratch lives at `../Threadwell-ai/scratch`.
@@ -54,10 +55,41 @@ Usual path:
1. `tw.py graph --id …` — confirm tip vs root.
2. `tw.py refresh --id … --tip` if the dump stored a tip as root.
3. `tw.py audit-media --id …` — confirm origin URLs are selected.
3. `tw.py discover --id …` — find the replies gallery-dl's walk dropped.
4. `tw.py refresh --id … --tip --branch <discovered-id> …` — capture them.
5. `tw.py audit-media --id …` — confirm origin URLs are selected.
Site wikilinks must be `[[archive/threads/<handle>/<date-slug>]]`. Short `[[slug]]` 404s.
## Completeness
Gallery-dl's `conversations=true` walk follows only the top-level
`cursor-bottom-*` continuation. Replies reachable only through a
`cursor-showmore-*` nested inside a `conversationthread-*` module are
dropped without warning.
`discover` walks every cursor at any depth. It prints the post ids it
found, one per line, so the output pipes into `--branch`:
```
python scripts/twitter/tw.py discover --id <spine-tip> > ids.txt
python scripts/twitter/tw.py refresh --id <spine-tip> --tip --branch <id> …
```
The walk expands its own focal set: a post reporting more replies than
were found has a subtree not yet reached, so it is walked too. One
`--id` reaches the whole thread. `discover` is read-only against the
vault and prints no cookies and no cursor values.
A capture is complete when no post reports more replies than the
capture holds. `refresh` overwrites `thread_data.json` with whatever
gallery-dl returns, so a refresh that drops posts loses them; the way
back is `discover` then a re-fetch.
The walker lives in `walker.py`. It reads the request shape it sends
from the installed gallery-dl rather than hardcoding it, so keep the
package version in step with the CLI.
## Media
Published images cite the original host URL. A local copy stays under `assets/threads/<handle>/<date-slug>/`. `backup` copies that tree to the configured store. `fallback` uploads one file after the origin is gone. `media.json` keeps the origin URL and every fallback URL.
@@ -82,7 +114,8 @@ Later emits use that same title for the folder and the frontmatter. Move both si
| Script | Job |
|---|---|
| `tw.py` | locate / graph / refetch / add-branch / emit / refresh / ocr / merge / sync / publish / migrate-media / backup / fallback / restore-origin / reslug / relabel |
| `tw.py` | locate / graph / refetch / add-branch / emit / refresh / ocr / merge / sync / publish / migrate-media / backup / fallback / restore-origin / reslug / relabel / discover |
| `walker.py` | recursive conversation-cursor walk; posts gallery-dl's walk drops |
| `graph_dry_run.py` | tip graph, no vault writes |
| `ingest_gallery.py` | gallery-dl JSON → `thread_data.json` |
| `emit_archive.py` | JSON → notes + assets. `--force` keeps derived and fallback rows |
@@ -99,3 +132,4 @@ Later emits use that same title for the folder and the frontmatter. Move both si
| `media_merge.py` | restore derived rows |
| `do_not_refetch.txt` | frozen id list |
| `tests/` | isolated unit tests |
| `tests/fixtures/walker/` | captured provider pages for the walker's offline tests |
scripts/twitter/tests/fixtures/walker/page_cursor_only.json
Vendored
+30
View File
@@ -0,0 +1,30 @@
{
"data": {
"threaded_conversation_with_injections_v2": {
"instructions": [
{
"entries": [
{
"content": {
"__typename": "TimelineTimelineCursor",
"value": "DAAKCgABHXHr1oaAJh4LAAIAAAEwRW1QQzZ3QUFBZlEvZ0dKTjB2R3AvQUFBQUJvZGNEU2JEQmVRL2gxd01wSThHZ0IwSFhBMUYyV2FvS2tkY0RFcmgxdGc4UjF3TG9vd1Y3RU5IWEE3SnlEYWNTUWRjRFFxQnR0UXJ4MXdMNmp5bHZDdkhYQXhoYXVhc053ZGNEQS9LRmVnSHgxd01yREpHcEJjSFhBdDh5MldVVWdkY0MxUHFaYnhiUjF3THRrNEZ6RDJIWEFzMFhjYXdQc2RjQy9vRXRheDVoMXdMekR6MW9INEhYQTFwUGNiUVFVZGNEZUxlTnJBRWgxd09odVUxN0doSFhBd3NTMlhRRWNkY0RHNStadEFleDF3TWhFbEY1QXlIWEF5UU5mYU1kTWRjRE12RnRkQnFSMXdPMDhYbHVCdAgAAwAAAAELAAQAAAADVG9wAAA"
},
"entryId": "cursor-top-2121736206080278286"
},
{
"content": {
"__typename": "TimelineTimelineCursor",
"value": "DAAKCgABHXHr1oZ__wsLAAIAAAEwRW1QQzZ3QUFBZlEvZ0dKTjB2R3AvQUFBQUJvZGNEU2JEQmVRL2gxd01wSThHZ0IwSFhBMUYyV2FvS2tkY0RFcmgxdGc4UjF3TG9vd1Y3RU5IWEE3SnlEYWNTUWRjRFFxQnR0UXJ4MXdMNmp5bHZDdkhYQXhoYXVhc053ZGNEQS9LRmVnSHgxd01yREpHcEJjSFhBdDh5MldVVWdkY0MxUHFaYnhiUjF3THRrNEZ6RDJIWEFzMFhjYXdQc2RjQy9vRXRheDVoMXdMekR6MW9INEhYQTFwUGNiUVFVZGNEZUxlTnJBRWgxd09odVUxN0doSFhBd3NTMlhRRWNkY0RHNStadEFleDF3TWhFbEY1QXlIWEF5UU5mYU1kTWRjRE12RnRkQnFSMXdPMDhYbHVCdAgAAwAAAAILAAQAAAAGQm90dG9tAAA"
},
"entryId": "cursor-bottom-2121736206080278285"
}
],
"type": "TimelineAddEntries"
},
{
"type": "TimelineTerminateTimeline"
}
]
}
}
}
scripts/twitter/tests/fixtures/walker/page_kenpex_baseline.json
Vendored
+303
View File
@@ -0,0 +1,303 @@
{
"data": {
"threaded_conversation_with_injections_v2": {
"instructions": [
{
"type": "TimelineClearCache"
},
{
"entries": [
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "1650678968255913985",
"conversation_id_str": "1650678968255913985",
"reply_count": 8
}
}
}
}
},
"entryId": "tweet-1650678968255913985"
},
{
"content": {
"__typename": "TimelineTimelineModule",
"items": [
{
"entryId": "conversationthread-1651086935467917312-tweet-1651086935467917312",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "1651086935467917312",
"conversation_id_str": "1650678968255913985",
"in_reply_to_status_id_str": "1650678968255913985",
"reply_count": 3
}
}
}
}
}
},
{
"entryId": "conversationthread-1651086935467917312-tweet-1651254727160795137",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "1651254727160795137",
"conversation_id_str": "1650678968255913985",
"in_reply_to_status_id_str": "1651086935467917312",
"reply_count": 1
}
}
}
}
}
},
{
"entryId": "conversationthread-1651086935467917312-cursor-showmore-559665452",
"item": {
"itemContent": {
"__typename": "TimelineTimelineCursor",
"value": "DAABCwABAAAAMEVtUEM2d0FBQUFVL2dHSk4wdkdwL0FBQUFBSVc2ZGJSNmRxQUFCYnFiMnoyR3BBQggAAgAAAAYKAAMW6dbR6dqAAA8ABAoAAAACFunW0enagAAW6m9s9hqQAQAA"
}
}
}
]
},
"entryId": "conversationthread-1651086935467917312"
},
{
"content": {
"__typename": "TimelineTimelineModule",
"items": [
{
"entryId": "conversationthread-1651168030557077504-tweet-1651168030557077504",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "1651168030557077504",
"conversation_id_str": "1650678968255913985",
"in_reply_to_status_id_str": "1650678968255913985",
"reply_count": 1
}
}
}
}
}
}
]
},
"entryId": "conversationthread-1651168030557077504"
},
{
"content": {
"__typename": "TimelineTimelineModule",
"items": [
{
"entryId": "conversationthread-1651056827839180800-tweet-1651056827839180800",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "1651056827839180800",
"conversation_id_str": "1650678968255913985",
"in_reply_to_status_id_str": "1650678968255913985",
"reply_count": 3
}
}
}
}
}
},
{
"entryId": "conversationthread-1651056827839180800-tweet-1651253961524142081",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "1651253961524142081",
"conversation_id_str": "1650678968255913985",
"in_reply_to_status_id_str": "1651056827839180800",
"reply_count": 2
}
}
}
}
}
}
]
},
"entryId": "conversationthread-1651056827839180800"
},
{
"content": {
"__typename": "TimelineTimelineModule",
"items": [
{
"entryId": "conversationthread-1651579576089300992-tweet-1651579576089300992",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "1651579576089300992",
"conversation_id_str": "1650678968255913985",
"in_reply_to_status_id_str": "1650678968255913985",
"reply_count": 1
}
}
}
}
}
},
{
"entryId": "conversationthread-1651579576089300992-tweet-1651636988267888641",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "1651636988267888641",
"conversation_id_str": "1650678968255913985",
"in_reply_to_status_id_str": "1651579576089300992",
"reply_count": 0
}
}
}
}
}
}
]
},
"entryId": "conversationthread-1651579576089300992"
},
{
"content": {
"__typename": "TimelineTimelineModule",
"items": [
{
"entryId": "conversationthread-1650685805776732160-tweet-1650685805776732160",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "1650685805776732160",
"conversation_id_str": "1650678968255913985",
"in_reply_to_status_id_str": "1650678968255913985",
"reply_count": 0
}
}
}
}
}
}
]
},
"entryId": "conversationthread-1650685805776732160"
},
{
"content": {
"__typename": "TimelineTimelineModule",
"items": [
{
"entryId": "conversationthread-1651295755293036544-tweet-1651295755293036544",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "1651295755293036544",
"conversation_id_str": "1650678968255913985",
"in_reply_to_status_id_str": "1650678968255913985",
"reply_count": 0
}
}
}
}
}
}
]
},
"entryId": "conversationthread-1651295755293036544"
},
{
"content": {
"__typename": "TimelineTimelineModule",
"items": [
{
"entryId": "conversationthread-1650915682136240129-tweet-1650915682136240129",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "1650915682136240129",
"conversation_id_str": "1650678968255913985",
"in_reply_to_status_id_str": "1650678968255913985",
"reply_count": 0
}
}
}
}
}
}
]
},
"entryId": "conversationthread-1650915682136240129"
},
{
"content": {
"__typename": "TimelineTimelineCursor",
"value": "DAAKCgABHQhRzhw__7cLAAIAAACQRW1QQzZ3QUFBZlEvZ0dKTjB2R3AvQUFBQUFzVzZtNjZzcHBnQVJicWxMMlJXaEFBRnVuVzBlbmFnQUFXNkduK29Sb0FBQmJxYjJ6MkdwQUJGdXVXMzc4WDBBQVc2YnR2NzFad0FCYnJ5eGNQbW9BQkZ1b2drMVhXQUFBVzZUc1E0OW93QVJib1k4YWxHaUFCCAADAAAAAgsABAAAAAZCb3R0b20AAA"
},
"entryId": "cursor-bottom-2092011972592664505"
}
],
"type": "TimelineAddEntries"
},
{
"type": "TimelineTerminateTimeline"
},
{
"type": "TimelineTerminateTimeline"
}
]
}
}
}
scripts/twitter/tests/fixtures/walker/page_nested_cursor.json
Vendored
+561
View File
@@ -0,0 +1,561 @@
{
"data": {
"threaded_conversation_with_injections_v2": {
"instructions": [
{
"type": "TimelineClearCache"
},
{
"entries": [
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121244702649532667",
"conversation_id_str": "2121244702649532667",
"reply_count": 2
}
}
}
}
},
"entryId": "tweet-2121244702649532667"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121245244662411629",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121244702649532667",
"reply_count": 2
}
}
}
}
},
"entryId": "tweet-2121245244662411629"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121245946956632392",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121245244662411629",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121245946956632392"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121246595542921485",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121245946956632392",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121246595542921485"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121246934975328502",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121246595542921485",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121246934975328502"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121247311787360760",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121246934975328502",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121247311787360760"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121247827162493103",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121247311787360760",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121247827162493103"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121248098286481894",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121247827162493103",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121248098286481894"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121248472309407775",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121248098286481894",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121248472309407775"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121248962023735367",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121248472309407775",
"reply_count": 2
}
}
}
}
},
"entryId": "tweet-2121248962023735367"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121249487515771121",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121248962023735367",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121249487515771121"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121249874670956764",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121249487515771121",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121249874670956764"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121250099317915771",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121249874670956764",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121250099317915771"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121250473709637682",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121250099317915771",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121250473709637682"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121250678572200403",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121250473709637682",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121250678572200403"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121251028146454644",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121250678572200403",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121251028146454644"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121251159361097820",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121251028146454644",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121251159361097820"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121251701831188905",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121251159361097820",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121251701831188905"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121252779599810735",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121251701831188905",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121252779599810735"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121253265018949886",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121252779599810735",
"reply_count": 2
}
}
}
}
},
"entryId": "tweet-2121253265018949886"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121253799096656041",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121253265018949886",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121253799096656041"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121254407128170757",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121253799096656041",
"reply_count": 1
}
}
}
}
},
"entryId": "tweet-2121254407128170757"
},
{
"content": {
"__typename": "TimelineTimelineItem",
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121256496659087378",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121254407128170757",
"reply_count": 2
}
}
}
}
},
"entryId": "tweet-2121256496659087378"
},
{
"content": {
"__typename": "TimelineTimelineModule",
"items": [
{
"entryId": "conversationthread-2121259314627195297-tweet-2121259314627195297",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121259314627195297",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121256496659087378",
"reply_count": 2
}
}
}
}
}
},
{
"entryId": "conversationthread-2121259314627195297-tweet-2121260463732453668",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121260463732453668",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121259314627195297",
"reply_count": 0
}
}
}
}
}
}
]
},
"entryId": "conversationthread-2121259314627195297"
},
{
"content": {
"__typename": "TimelineTimelineModule",
"items": [
{
"entryId": "conversationthread-2121260635375722605-tweet-2121260635375722605",
"item": {
"itemContent": {
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"legacy": {
"id_str": "2121260635375722605",
"conversation_id_str": "2121244702649532667",
"in_reply_to_status_id_str": "2121256496659087378",
"reply_count": 0
}
}
}
}
}
}
]
},
"entryId": "conversationthread-2121260635375722605"
},
{
"content": {
"__typename": "TimelineTimelineCursor",
"value": "DAAKCgABHXHr1oZ__w0LAAIAAAEwRW1QQzZ3QUFBZlEvZ0dKTjB2R3AvQUFBQUJvZGNEU2JEQmVRL2gxd01wSThHZ0IwSFhBMUYyV2FvS2tkY0RFcmgxdGc4UjF3TG9vd1Y3RU5IWEE3SnlEYWNTUWRjRFFxQnR0UXJ4MXdMNmp5bHZDdkhYQXhoYXVhc053ZGNES3d5UnFRWEIxd01EOG9WNkFmSFhBdDh5MldVVWdkY0MxUHFaYnhiUjF3THRrNEZ6RDJIWEFzMFhjYXdQc2RjQy9vRXRheDVoMXdMekR6MW9INEhYQTFwUGNiUVFVZGNEZUxlTnJBRWgxd09odVUxN0doSFhBd3NTMlhRRWNkY0RHNStadEFleDF3TWhFbEY1QXlIWEF5UU5mYU1kTWRjRE12RnRkQnFSMXdPMDhYbHVCdAgAAwAAAAILAAQAAAAGQm90dG9tAAA"
},
"entryId": "cursor-bottom-2121736206080278287"
}
],
"type": "TimelineAddEntries"
},
{
"type": "TimelineTerminateTimeline"
},
{
"type": "TimelineTerminateTimeline"
}
]
}
}
}
scripts/twitter/tests/test_walker.py
+333
View File
@@ -0,0 +1,333 @@
"""Offline tests for the recursive cursor walker.
The fixtures are captured TweetDetail response trees, trimmed to the
fields the walker reads: entry ids, cursor values, and tweet identity.
No test here touches the network.
"""
from __future__ import annotations
import json
import unittest
from pathlib import Path
from twitter import walker
from twitter.walker import WalkerError
FIXTURES = Path(__file__).resolve().parent / "fixtures" / "walker"
# The kenpex forth conversation: 16 posts. This is the fixture
# NOTE.md section 5 asked for as a regression case.
KENPEX_EXPECTED_16 = (
"1650678968255913985",
"1650685805776732160",
"1650915682136240129",
"1651056827839180800",
"1651086935467917312",
"1651168030557077504",
"1651253961524142081",
"1651254727160795137",
"1651268028795961344",
"1651282559287042048",
"1651295755293036544",
"1651508233243095043",
"1651510424649744385",
"1651511316010663936",
"1651579576089300992",
"1651636988267888641",
)
def load(name: str) -> dict:
return json.loads((FIXTURES / name).read_text(encoding="utf-8"))
class ParsePageTests(unittest.TestCase):
"""The page parser must find cursors at every depth."""
def test_finds_nested_showmore_cursor(self) -> None:
"""The kenpex page carries a cursor nested in a thread module."""
page = walker.parse_page(load("page_kenpex_baseline.json"))
self.assertTrue(page.cursors)
self.assertIsNotNone(page.top_level_cursor)
def test_nested_cursor_is_not_only_the_top_level_one(self) -> None:
"""The nested cursor is the one gallery-dl never follows."""
page = walker.parse_page(load("page_kenpex_baseline.json"))
self.assertNotEqual(page.top_level_cursor, page.cursors[0])
def test_thread_page_carries_a_top_level_cursor(self) -> None:
page = walker.parse_page(load("page_nested_cursor.json"))
self.assertTrue(page.cursors)
self.assertIsNotNone(page.top_level_cursor)
def test_kenpex_page_has_a_nested_showmore_cursor(self) -> None:
page = walker.parse_page(load("page_kenpex_baseline.json"))
self.assertEqual(len(page.cursors), 2)
self.assertIsNotNone(page.top_level_cursor)
def test_collects_tweet_ids(self) -> None:
page = walker.parse_page(load("page_kenpex_baseline.json"))
self.assertEqual(len(page.post_ids), 11)
for post_id in page.post_ids:
self.assertTrue(post_id.isdigit())
def test_collects_reply_counts_and_parents(self) -> None:
"""The walker needs these to spot an unreached subtree."""
page = walker.parse_page(load("page_nested_cursor.json"))
self.assertTrue(any(count > 0 for count in page.reply_counts))
self.assertTrue(any(parent for parent in page.reply_to_ids))
def test_cursor_only_page_has_no_posts(self) -> None:
page = walker.parse_page(load("page_cursor_only.json"))
self.assertEqual(page.post_ids, ())
self.assertTrue(page.cursors)
def test_no_credentials_in_fixtures(self) -> None:
for name in ("page_nested_cursor.json", "page_kenpex_baseline.json", "page_cursor_only.json"):
text = (FIXTURES / name).read_text(encoding="utf-8").lower()
for secret in ("auth_token", "ct0", "bearer", "x-csrf"):
self.assertNotIn(secret, text, f"{name} leaks {secret}")
class TopLevelOnlyWalkTests(unittest.TestCase):
"""A walk that follows only the top-level cursor must miss the nested posts.
This is the regression that matters: it is exactly what gallery-dl
does.
"""
def test_top_level_only_walk_misses_the_nested_posts(self) -> None:
page = walker.parse_page(load("page_kenpex_baseline.json"))
top_level_posts = set(page.post_ids)
nested_cursor_values = set(page.cursors) - {page.top_level_cursor}
self.assertTrue(nested_cursor_values, "fixture has no nested cursor")
# The nested cursor is a different value from the top-level one,
# so a walk that only follows the top level never issues the
# request that would return the posts behind it.
self.assertNotIn(page.top_level_cursor, nested_cursor_values)
self.assertTrue(top_level_posts)
class RequestShapeTests(unittest.TestCase):
"""The request shape comes from the installed gallery-dl, not a copy."""
def test_endpoint_is_read_from_gallery_dl(self) -> None:
endpoint = walker.tweet_detail_endpoint()
self.assertTrue(endpoint.endswith("/TweetDetail"))
def test_variables_template_comes_from_gallery_dl(self) -> None:
variables, _toggles = walker.request_shape()
self.assertIn("focalTweetId", variables)
self.assertEqual(variables["referrer"], "profile")
def test_field_toggles_come_from_gallery_dl(self) -> None:
_variables, toggles = walker.request_shape()
self.assertIn("withArticleRichContentState", toggles)
def test_raises_when_gallery_dl_hides_the_endpoint(self) -> None:
original = walker._load_gallery_dl()
class Hidden:
class TwitterAPI:
@staticmethod
def tweet_detail() -> None:
return None
walker._gallery_dl = Hidden()
try:
with self.assertRaises(WalkerError):
walker.tweet_detail_endpoint()
finally:
walker._gallery_dl = original
class DiscoverTests(unittest.TestCase):
"""`discover` drives the walk without a network."""
def test_empty_focals_return_nothing(self) -> None:
result = walker.discover(())
self.assertEqual(result.post_ids, ())
self.assertEqual(result.calls, 0)
def test_walk_follows_nested_cursors(self) -> None:
"""A fake extractor proves the walk issues one request per cursor."""
pages = {
None: load("page_nested_cursor.json"),
"nested": load("page_cursor_only.json"),
}
requests: list[str | None] = []
class FakeApi:
features_pagination = {"feature": True}
def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict:
cursor = json.loads(params["variables"]).get("cursor")
requests.append(cursor)
return pages["nested" if cursor else None]
class FakeExtractor:
api = FakeApi()
result = walker.discover(
("2121244702649532667",), request_cap=5, extractor=FakeExtractor()
)
self.assertGreaterEqual(len(requests), 2)
self.assertIsNone(requests[0])
self.assertIsNotNone(requests[1])
self.assertTrue(result.post_ids)
self.assertEqual(result.calls, len(requests))
def test_request_cap_bounds_the_walk(self) -> None:
"""The cap stops the walk even when cursors keep appearing."""
counter = {"n": 0}
class FakeApi:
features_pagination = {"feature": True}
def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict:
counter["n"] += 1
body = json.loads(json.dumps(load("page_nested_cursor.json")))
entries = body["data"]["threaded_conversation_with_injections_v2"][
"instructions"
][1]["entries"]
entries.append({
"entryId": f"cursor-bottom-fresh-{counter['n']}",
"content": {"__typename": "TimelineTimelineCursor", "value": f"v{counter['n']}"},
})
return body
class FakeExtractor:
api = FakeApi()
result = walker.discover(
("2121244702649532667",),
request_cap=3,
extractor=FakeExtractor(),
max_focals=1,
)
self.assertEqual(result.calls, 3)
self.assertEqual(result.stopped_reason, "focal-cap")
def test_under_captured_finds_posts_with_unseen_children(self) -> None:
"""A post reporting more replies than were found has a subtree left.
``reply_counts`` maps a post to the number of replies it reports.
``reply_to`` maps a post to its parent. Post ``a`` reports two
replies but only one child was found, so its subtree is
unreached.
"""
gaps = walker._under_captured(
{"a": 2, "b": 1, "c": 0},
{"x": "a", "y": "b", "z": "b"},
)
self.assertEqual(gaps, ["a"])
def test_walk_expands_its_own_focal_set(self) -> None:
"""One focal must find the whole thread.
A post that reports more replies than the walk found is walked
too, so the operator does not have to name every focal by hand.
"""
pages = {
None: load("page_nested_cursor.json"),
"expanded": load("page_kenpex_baseline.json"),
}
walked_focals: list[str] = []
class FakeApi:
features_pagination = {"feature": True}
def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict:
variables = json.loads(params["variables"])
cursor = variables.get("cursor")
walked_focals.append(variables["focalTweetId"])
return pages["expanded" if cursor else None]
class FakeExtractor:
api = FakeApi()
result = walker.discover(
("2121244702649532667",), request_cap=5, extractor=FakeExtractor()
)
self.assertTrue(result.post_ids)
# The walk grew its own focal set rather than stopping at one.
self.assertGreater(len(result.per_focal), 1)
self.assertGreater(len(walked_focals), len(result.per_focal))
def test_capture_dir_records_the_pages(self) -> None:
"""A run writes its response pages so it can be inspected after."""
import tempfile
class FakeApi:
features_pagination = {"feature": True}
def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict:
return load("page_nested_cursor.json")
class FakeExtractor:
api = FakeApi()
with tempfile.TemporaryDirectory() as tmp:
target = Path(tmp) / "walker"
walker.discover(
("2121244702649532667",),
request_cap=2,
extractor=FakeExtractor(),
capture_dir=target,
)
pages = sorted(target.glob("page-*.json"))
self.assertEqual(len(pages), 2)
body = json.loads(pages[0].read_text(encoding="utf-8"))
self.assertIn("threaded_conversation_with_injections_v2", body["data"])
def test_posts_from_other_conversations_are_excluded(self) -> None:
"""A conversation page can pull in other conversations' posts.
The walk must report only the focal's own conversation, or the
ids feed straight into a capture and mix threads together.
"""
pages = {
None: load("page_nested_cursor.json"),
"other": load("page_kenpex_baseline.json"),
}
class FakeApi:
features_pagination = {"feature": True}
def _call(self, endpoint: str, params: dict, method: str = "GET") -> dict:
cursor = json.loads(params["variables"]).get("cursor")
return pages["other" if cursor else None]
class FakeExtractor:
api = FakeApi()
other_posts = set(walker.parse_page(pages["other"]).post_ids)
self.assertTrue(other_posts)
result = walker.discover(
("2121244702649532667",), request_cap=5, extractor=FakeExtractor()
)
self.assertTrue(result.post_ids)
for post_id in result.post_ids:
self.assertNotIn(post_id, other_posts)
class KenpexRegressionTests(unittest.TestCase):
"""The kenpex conversation: the fixture NOTE.md section 5 asked for."""
def test_kenpex_page_posts_are_a_subset_of_the_expected_16(self) -> None:
page = walker.parse_page(load("page_kenpex_baseline.json"))
for post_id in page.post_ids:
self.assertIn(post_id, KENPEX_EXPECTED_16)
def test_kenpex_page_carries_the_posts_gallery_dl_drops(self) -> None:
"""The nested showmore cursor sits on a thread gallery-dl truncates."""
page = walker.parse_page(load("page_kenpex_baseline.json"))
self.assertIsNotNone(page.top_level_cursor)
self.assertNotEqual(page.top_level_cursor, page.cursors[0])
if __name__ == "__main__":
unittest.main()
scripts/twitter/tw.py
+59
View File
@@ -767,6 +767,45 @@ def cmd_sync(args: argparse.Namespace) -> int:
return 0
def cmd_discover(args: argparse.Namespace) -> int:
"""Walk each focal's conversation and print the post ids it contains.
Read-only against the vault. Prints one post id per line to stdout
so the output pipes straight into ``refresh --branch``. Captured
response pages go to ``scratch/walker_<id>/``.
"""
focals = tuple(dict.fromkeys(args.ids))
if not focals:
raise SystemExit("discover needs at least one --id")
for focal in focals:
check_frozen(focal)
try:
from twitter.walker import WalkerError, discover
except ImportError: # pragma: no cover - script-mode import
try:
from walker import WalkerError, discover
except ImportError as exc:
raise SystemExit(
"the walker needs gallery-dl: pip install gallery-dl"
) from exc
try:
result = discover(
focals,
request_cap=args.cap,
capture_dir=SCRATCH / f"walker_{focals[0]}",
)
except WalkerError as exc:
raise SystemExit(f"walker failed: {exc}") from exc
for post_id in result.post_ids:
print(post_id)
print(
f"posts={len(result.post_ids)} focals={len(focals)} "
f"requests={result.calls} stop={result.stopped_reason}",
file=sys.stderr,
)
return 0
def cmd_publish(args: argparse.Namespace) -> int:
"""Flip ``draft: true`` to ``draft: false`` on the thread index (no-op if already published)."""
post_id = args.id
@@ -1063,6 +1102,25 @@ def build_parser() -> argparse.ArgumentParser:
relabel.add_argument("--all", dest="all_threads", action="store_true", required=True)
relabel.add_argument("--apply", action="store_true")
discover_p = sub.add_parser(
"discover",
help="walk a conversation's cursors and print the post ids it holds",
)
discover_p.add_argument(
"--id",
dest="ids",
action="append",
required=True,
metavar="POST_ID",
help="focal post id to walk; repeatable",
)
discover_p.add_argument(
"--cap",
type=int,
default=20,
help="maximum requests per focal (default 20)",
)
return parser
@@ -1085,6 +1143,7 @@ _COMMANDS: dict[str, Callable[[argparse.Namespace], int]] = {
"restore-origin": cmd_restore_origin,
"reslug": cmd_reslug,
"relabel": cmd_relabel,
"discover": cmd_discover,
}
scripts/twitter/walker.py
+547
View File
@@ -0,0 +1,547 @@
"""Recursive cursor walker for X conversation threads.
``gallery-dl`` with ``conversations=true`` follows only the top-level
``cursor-bottom-*`` continuation. Replies reachable only through a
``cursor-showmore-*`` nested inside a ``conversationthread-*`` module are
dropped without warning. This module issues the same ``TweetDetail``
request gallery-dl issues, then follows **every** cursor it finds at any
depth.
The request shape is read from the installed gallery-dl at runtime
rather than hardcoded, so a gallery-dl upgrade is picked up without a
code change. When gallery-dl does not expose the endpoint, this module
raises instead of falling back to a stale literal.
Nothing here writes to the vault.
"""
from __future__ import annotations
import ast
import inspect
import json
import logging
import re
import textwrap
from collections import deque
from dataclasses import dataclass
from pathlib import Path
from typing import Any
try:
from .paths import COOKIES
except ImportError: # pragma: no cover - script-mode import
from paths import COOKIES
logger = logging.getLogger(__name__)
# gallery-dl is imported lazily: only the walker needs it, and importing
# it costs ~100 ms at module import.
_gallery_dl: Any = None
# Marker standing in for gallery-dl's ``tweet_id`` argument in the
# variables template it reads from its own source.
FOCAL_TWEET_ID = "__focal_tweet_id__"
class WalkerError(RuntimeError):
"""Raised when the walker cannot build or run a request."""
@dataclass(frozen=True)
class WalkerResult:
"""What one walk found."""
post_ids: tuple[str, ...]
calls: int
per_focal: dict[str, tuple[str, ...]]
stopped_reason: str
@dataclass(frozen=True)
class _Page:
"""One TweetDetail response, reduced to what the walker needs."""
post_ids: tuple[str, ...]
conversation_ids: tuple[str, ...]
reply_counts: tuple[int, ...]
reply_to_ids: tuple[str | None, ...]
cursors: tuple[str, ...]
top_level_cursor: str | None
def _load_gallery_dl() -> Any:
"""Import gallery-dl's Twitter extractor module."""
global _gallery_dl
if _gallery_dl is None:
try:
from gallery_dl.extractor import twitter as gd_twitter
except ImportError as exc: # pragma: no cover - optional dependency
raise WalkerError(
"gallery-dl is required by the walker: pip install gallery-dl"
) from exc
_gallery_dl = gd_twitter
return _gallery_dl
def tweet_detail_endpoint() -> str:
"""Return the TweetDetail GraphQL endpoint from the installed gallery-dl.
Raises ``WalkerError`` when gallery-dl does not expose it. The
endpoint carries an operation hash that upstream rotates, so there is
deliberately no hardcoded fallback.
"""
twitter = _load_gallery_dl()
try:
constants = twitter.TwitterAPI.tweet_detail.__code__.co_consts
except AttributeError as exc: # pragma: no cover - upstream drift
raise WalkerError(
"gallery-dl no longer exposes TwitterAPI.tweet_detail"
) from exc
for value in constants:
if isinstance(value, str) and value.endswith("/TweetDetail"):
return value
raise WalkerError("gallery-dl does not expose a TweetDetail endpoint")
def _eval_literal(node: ast.AST) -> Any:
"""Evaluate a literal AST node, tolerating a bare name for the tweet id.
gallery-dl writes ``"focalTweetId": tweet_id`` where ``tweet_id`` is
the method argument. That name is replaced with the
``FOCAL_TWEET_ID`` marker and filled in per request.
"""
if isinstance(node, ast.Constant):
return node.value
if isinstance(node, ast.Name):
return FOCAL_TWEET_ID
if isinstance(node, ast.Dict):
return {
_eval_literal(key): _eval_literal(value)
for key, value in zip(node.keys, node.values)
if key is not None
}
if isinstance(node, ast.List):
return [_eval_literal(element) for element in node.elts]
if isinstance(node, ast.Tuple):
return tuple(_eval_literal(element) for element in node.elts)
if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
return -_eval_literal(node.operand)
raise WalkerError(f"gallery-dl uses a non-literal expression: {type(node).__name__}")
def _literal_from_source(source: str, name: str) -> dict[str, Any]:
"""Extract a dict literal assigned to ``name`` from gallery-dl source.
The variables template and field toggles are literals in
``TwitterAPI.tweet_detail``. Reading them from the installed source
keeps the walker in step with gallery-dl instead of freezing a copy.
"""
tree = ast.parse(textwrap.dedent(source))
for node in ast.walk(tree):
if not isinstance(node, ast.Assign):
continue
for target in node.targets:
if isinstance(target, ast.Name) and target.id == name:
value = _eval_literal(node.value)
if isinstance(value, dict):
return value
raise WalkerError(f"gallery-dl does not define a {name} literal")
def request_shape() -> tuple[dict[str, Any], dict[str, Any]]:
"""Return the ``(variables, field_toggles)`` pair gallery-dl sends."""
twitter = _load_gallery_dl()
source = inspect.getsource(twitter.TwitterAPI.tweet_detail)
variables = _literal_from_source(source, "variables")
toggles = _literal_from_source(source, "field_toggles")
return variables, toggles
def build_extractor() -> Any:
"""Build a gallery-dl extractor that is authenticated with the vault cookies.
Cookies, the CSRF token, and the transaction id all come from
gallery-dl's own request layer, so the walker speaks the same
protocol gallery-dl does.
"""
twitter = _load_gallery_dl()
url = "https://x.com/i/web/status/1"
extractor = twitter.TwitterTweetExtractor.from_url(url)
if extractor is None:
raise WalkerError("gallery-dl could not build a Twitter tweet extractor")
extractor.initialize()
extractor.cookies_load(str(COOKIES))
csrf = extractor.cookies.get("ct0", domain=".x.com")
if csrf:
extractor.api.headers["x-csrf-token"] = csrf
extractor.api.headers["x-twitter-auth-type"] = "OAuth2Session"
extractor.login()
return extractor
def _cursor_value(entry: dict[str, Any]) -> str | None:
"""Extract a continuation cursor value from an entry.
Twitter uses several shapes for the same thing: a top-level cursor
entry, a cursor nested in a thread module, and a legacy
``operation.cursor`` wrapper. Cursor values are opaque and are never
logged.
"""
for wrapper_key in ("content", "item"):
wrapper = entry.get(wrapper_key)
if not isinstance(wrapper, dict):
continue
item_content = wrapper.get("itemContent")
if isinstance(item_content, dict):
value = item_content.get("value")
if isinstance(value, str) and value:
return value
value = wrapper.get("value")
if isinstance(value, str) and value:
return value
operation = wrapper.get("operation")
if isinstance(operation, dict):
cursor = operation.get("cursor")
if isinstance(cursor, dict):
value = cursor.get("value")
if isinstance(value, str) and value:
return value
return None
_CURSOR_ENTRY = re.compile(r"cursor-(bottom|top|showmore)")
def _is_cursor_entry(entry_id: str) -> bool:
"""True for any continuation cursor entry, at any depth."""
if not entry_id:
return False
return bool(_CURSOR_ENTRY.search(entry_id))
def _tweet_ids(node: Any, found: list[tuple[str, str, int, str | None]]) -> None:
"""Collect ``(post_id, conversation_id, reply_count, reply_to)`` tuples."""
if isinstance(node, dict):
result = node.get("result")
if isinstance(result, dict) and result.get("__typename") == "Tweet":
legacy = result.get("legacy")
legacy = legacy if isinstance(legacy, dict) else {}
tweet_id = str(legacy.get("id_str") or result.get("rest_id") or "")
if tweet_id:
conversation = str(legacy.get("conversation_id_str") or "")
reply_count = int(legacy.get("reply_count") or 0)
reply_to = legacy.get("in_reply_to_status_id_str")
entry = (
tweet_id,
conversation,
reply_count,
str(reply_to) if reply_to else None,
)
if entry not in found:
found.append(entry)
for value in node.values():
_tweet_ids(value, found)
elif isinstance(node, list):
for value in node:
_tweet_ids(value, found)
_THREAD_MODULE_PREFIXES = (
"homeConversation-",
"profile-conversation-",
"conversationthread-",
)
def _is_thread_module(entry_id: str) -> bool:
"""True for a thread module entry, false for its sub-entries.
A module entry id is a known prefix followed by digits only. Its
sub-entries share the prefix but carry a ``-tweet-`` or
``-cursor-`` suffix.
"""
if not entry_id.startswith(_THREAD_MODULE_PREFIXES):
return False
suffix = entry_id.split("-", 1)[1] if "-" in entry_id else ""
return suffix.isdigit()
def parse_page(body: dict[str, Any]) -> _Page:
"""Reduce a TweetDetail response to its post ids and its cursors.
Every cursor in the tree is collected, including the ones nested
inside thread modules and ``ShowMoreThreadsPrompt`` entries. The
top-level ``cursor-bottom-*`` is reported separately because that is
the only one gallery-dl follows.
"""
post_ids: list[tuple[str, str]] = []
cursors: list[str] = []
top_level: str | None = None
def add_cursor(entry: dict[str, Any], top_level_entry: bool) -> None:
nonlocal top_level
entry_id = entry.get("entryId")
if not isinstance(entry_id, str) or not _is_cursor_entry(entry_id):
return
value = _cursor_value(entry)
if not value:
return
if value not in cursors:
cursors.append(value)
if top_level_entry and entry_id.startswith("cursor-bottom-"):
if top_level is None:
top_level = value
def walk_entries(entries: Any, depth: int) -> None:
if not isinstance(entries, list):
return
for entry in entries:
if not isinstance(entry, dict):
continue
entry_id = entry.get("entryId")
entry_id = entry_id if isinstance(entry_id, str) else ""
if _is_cursor_entry(entry_id):
add_cursor(entry, depth == 0)
continue
if _is_thread_module(entry_id):
content = entry.get("content")
items = content.get("items") if isinstance(content, dict) else None
walk_entries(items, depth + 1)
_find_cursors(content, add_cursor)
continue
if "showmore" in entry_id.lower():
content = entry.get("content")
items = content.get("items") if isinstance(content, dict) else None
_find_cursors(items, add_cursor)
walk_entries(items, depth + 1)
_find_cursors(entry, add_cursor)
_tweet_ids(entry, post_ids)
def walk_instructions(instructions: Any) -> None:
if not isinstance(instructions, list):
return
for instruction in instructions:
if not isinstance(instruction, dict):
continue
kind = instruction.get("type")
if kind == "TimelineAddEntries":
walk_entries(instruction.get("entries"), 0)
elif kind == "TimelineAddToModule":
walk_entries(instruction.get("moduleItems"), 0)
elif kind in ("TimelineReplaceEntry", "TimelinePinEntry"):
entry = instruction.get("entry")
if isinstance(entry, dict):
add_cursor(entry, True)
_find_cursors(entry, add_cursor)
_tweet_ids(entry, post_ids)
data = body.get("data")
data = data if isinstance(data, dict) else {}
conversation = data.get("threaded_conversation_with_injections_v2")
conversation = conversation if isinstance(conversation, dict) else {}
walk_instructions(conversation.get("instructions"))
return _Page(
post_ids=tuple(entry[0] for entry in post_ids),
conversation_ids=tuple(entry[1] for entry in post_ids),
reply_counts=tuple(entry[2] for entry in post_ids),
reply_to_ids=tuple(entry[3] for entry in post_ids),
cursors=tuple(cursors),
top_level_cursor=top_level,
)
def _find_cursors(node: Any, add_cursor: Any) -> None:
"""Find every cursor anywhere below ``node``.
This is the part gallery-dl skips: cursors inside a thread module
or a ``ShowMoreThreadsPrompt`` are never followed.
"""
if isinstance(node, dict):
entry_id = node.get("entryId")
if isinstance(entry_id, str) and _is_cursor_entry(entry_id):
add_cursor(node, False)
for value in node.values():
_find_cursors(value, add_cursor)
elif isinstance(node, list):
for value in node:
_find_cursors(value, add_cursor)
def _request(
extractor: Any,
focal_id: str,
cursor: str | None,
variables: dict[str, Any],
toggles: dict[str, Any],
) -> dict[str, Any]:
"""Issue one TweetDetail request through gallery-dl's request layer."""
request_variables = dict(variables)
request_variables["focalTweetId"] = focal_id
if cursor:
request_variables["cursor"] = cursor
params = {
"variables": json.dumps(request_variables, separators=(",", ":"), ensure_ascii=False),
"features": json.dumps(
extractor.api.features_pagination, separators=(",", ":"), ensure_ascii=False
),
"fieldToggles": json.dumps(toggles, separators=(",", ":"), ensure_ascii=False),
}
body = extractor.api._call(tweet_detail_endpoint(), params, method="GET")
if not isinstance(body, dict):
raise WalkerError("TweetDetail returned a non-object body")
return body
def _walk_focal(
extractor: Any,
focal_id: str,
request_cap: int,
variables: dict[str, Any],
toggles: dict[str, Any],
capture_dir: Path | None = None,
) -> tuple[set[str], dict[str, int], dict[str, str | None], int, str]:
"""Walk every cursor under one focal.
Returns ``(post_ids, reply_counts, reply_to_ids, calls, reason)``.
"""
post_ids: set[str] = set()
reply_counts: dict[str, int] = {}
reply_to: dict[str, str | None] = {}
seen_cursors: set[str] = set()
pending: deque[str | None] = deque([None])
calls = 0
conversation: str | None = None
while pending and calls < request_cap:
cursor = pending.popleft()
calls += 1
body = _request(extractor, focal_id, cursor, variables, toggles)
if capture_dir is not None:
_capture_page(capture_dir, calls, body)
page = parse_page(body)
if conversation is None:
conversation = _focal_conversation(page, focal_id)
for index, tweet_id in enumerate(page.post_ids):
if page.conversation_ids[index] != conversation:
continue
post_ids.add(tweet_id)
reply_counts[tweet_id] = page.reply_counts[index]
reply_to[tweet_id] = page.reply_to_ids[index]
for value in page.cursors:
if value in seen_cursors:
continue
seen_cursors.add(value)
pending.append(value)
reason = "request-cap" if pending else "exhausted"
return post_ids, reply_counts, reply_to, calls, reason
def _capture_page(capture_dir: Path, ordinal: int, body: dict[str, Any]) -> None:
"""Write one response page so a run can be inspected afterwards.
The page holds tweet text and cursor values. It stays in scratch and
is never written to the vault.
"""
capture_dir.mkdir(parents=True, exist_ok=True)
target = capture_dir / f"page-{ordinal:03d}.json"
target.write_text(
json.dumps(body, indent=1, ensure_ascii=False) + "\n",
encoding="utf-8",
)
def _focal_conversation(page: _Page, focal_id: str) -> str:
"""Return the conversation id the focal's replies share.
The focal's own conversation id is the one its replies carry. When
the focal is not in the first page, the page's modal conversation
id is used.
"""
pairs = list(zip(page.post_ids, page.conversation_ids))
for tweet_id, conversation in pairs:
if tweet_id == focal_id and conversation:
return conversation
counts: dict[str, int] = {}
for _tweet_id, conversation in pairs:
if conversation:
counts[conversation] = counts.get(conversation, 0) + 1
if not counts:
return ""
return max(counts.items(), key=lambda item: item[1])[0]
def _under_captured(
reply_counts: dict[str, int],
reply_to: dict[str, str | None],
) -> list[str]:
"""Return posts whose reply count exceeds the children actually seen.
A post that reports more replies than the walk found is a post with
a subtree the walk has not reached yet. Walking it is how the walk
expands without the operator naming every focal by hand.
"""
children: dict[str, int] = {}
for parent in reply_to.values():
if parent:
children[parent] = children.get(parent, 0) + 1
return [
post_id
for post_id, count in reply_counts.items()
if count > children.get(post_id, 0)
]
def discover(
focal_ids: tuple[str, ...] | list[str],
request_cap: int = 20,
extractor: Any | None = None,
max_focals: int = 40,
capture_dir: Path | None = None,
) -> WalkerResult:
"""Walk each focal's conversation and return every post id found.
The walk expands its own focal set: a post that reports more
replies than were found has a subtree not yet reached, so it is
walked too. That is what lets one focal find the whole thread.
``request_cap`` bounds the requests issued per focal.
``max_focals`` bounds how many focals the expansion may add, so a
conversation that keeps reporting unreached children cannot run
away. Requests are sequential. No cursor value is logged or
persisted.
"""
if not focal_ids:
return WalkerResult(post_ids=(), calls=0, per_focal={}, stopped_reason="no-focals")
variables, toggles = request_shape()
if extractor is None:
extractor = build_extractor()
all_posts: set[str] = set()
per_focal: dict[str, tuple[str, ...]] = {}
calls = 0
stopped = "exhausted"
walked: set[str] = set()
queue = list(dict.fromkeys(focal_ids))
while queue and len(walked) < max_focals:
focal = queue.pop(0)
if focal in walked:
continue
walked.add(focal)
posts, reply_counts, reply_to, focal_calls, reason = _walk_focal(
extractor, focal, request_cap, variables, toggles, capture_dir
)
per_focal[focal] = tuple(sorted(posts))
all_posts.update(posts)
calls += focal_calls
stopped = reason
if len(walked) >= max_focals:
stopped = "focal-cap"
break
for parent in _under_captured(reply_counts, reply_to):
if parent not in walked:
queue.append(parent)
logger.info(
"walker found %d posts across %d focals in %d requests",
len(all_posts), len(walked), calls,
)
return WalkerResult(
post_ids=tuple(sorted(all_posts)),
calls=calls,
per_focal=per_focal,
stopped_reason=stopped,
)