more branch handling commands (more sane).

scripts/twitter/README.md

@@ -7,6 +7,7 @@ From the vault root:
 ```
 python C:\projects\Threadwell\scripts\twitter\tw.py graph   --id <snowflake>
 python C:\projects\Threadwell\scripts\twitter\tw.py refresh --id <snowflake> --tip
+python C:\projects\Threadwell\scripts\twitter\tw.py refresh --id <spine-tip> --tip --branch <branch-tip>
 python C:\projects\Threadwell\scripts\twitter\tw.py audit-media --id <snowflake>
 python C:\projects\Threadwell\scripts\twitter\tw.py publish --id <snowflake>
 python C:\projects\Threadwell\scripts\twitter\tw.py sync    --handle <handle>
@@ -19,7 +20,7 @@ python C:\projects\Threadwell\scripts\twitter\tw.py migrate-media --all --apply
 
 `paths.py` sets vault = two parents above this file, dumps = `../manual_slop/docs/twitter`, scratch = `../Threadwell-ai/scratch`.
 
-`refresh` = refetch (cookies) + ingest + emit `--force` + media_merge. `--tip` treats `--id` as the tip and walks back to the OP regardless of handle. Each thread emits exactly one archive directory owned by the OP (the post with `reply_to_id == None`); cross-author responders do not get a directory. Does not flip `draft`. Does not commit. Frozen ids abort.
+`refresh` = refetch (cookies) + ingest + emit `--force` + media_merge. `--tip` treats `--id` as the tip and walks back to the OP regardless of handle. Repeatable `--branch <tip-id>` captures only those additional explicit tip paths and merges them into the same `thread_data.json`; it does not discover replies automatically. Captures run sequentially with gallery-dl retries disabled and five-second extractor/request pacing. Each thread emits exactly one archive directory owned by the OP (the post with `reply_to_id == None`); cross-author responders do not get a directory. Does not flip `draft`. Does not commit. Frozen ids abort.
 
 Usual publish path after you like the note:
 
@@ -62,4 +63,4 @@ Frozen threads (ids listed in `do_not_refetch.txt`, matched against every captur
 | `do_not_refetch.txt` | frozen id list |
 | `tests/` | isolated, deterministic unit tests |
 
-Secrets: `secrets/twitter_cookies.txt`, `secrets/credentials.toml` `[catbox] userhash` and `[backup] id`/`root`. The script reads them; never paste into chat, tests, or commits.
+Secrets: `secrets/twitter_cookies.txt`, `secrets/credentials.toml` `[catbox] userhash` and `[backup] id`/`root`. The script reads them; never paste into chat, tests, or commits.

scripts/twitter/tests/test_tw_media_commands.py

@@ -1,11 +1,87 @@
 from __future__ import annotations
 
+import json
 import unittest
+from pathlib import Path
+from tempfile import TemporaryDirectory
 
-from twitter.tw import build_parser
+from twitter.tw import (
+    _capture_ids,
+    _gallery_base_args,
+    _merge_gallery_files,
+    _validate_capture_ids,
+    build_parser,
+)
 
 
 class TwMediaCommandTests(unittest.TestCase):
+    def test_refetch_accepts_repeatable_branch(self) -> None:
+        args = build_parser().parse_args([
+            "refetch", "--id", "100",
+            "--branch", "200",
+            "--branch", "300",
+        ])
+        self.assertEqual(args.branch, ["200", "300"])
+
+    def test_refresh_accepts_repeatable_branch(self) -> None:
+        args = build_parser().parse_args([
+            "refresh", "--id", "100", "--tip",
+            "--branch", "200",
+        ])
+        self.assertEqual(args.branch, ["200"])
+
+    def test_capture_ids_deduplicates_in_first_seen_order(self) -> None:
+        self.assertEqual(
+            _capture_ids("100", ["200", "100", "300", "200"]),
+            ("100", "200", "300"),
+        )
+
+    def test_gallery_base_args_disable_retries_and_add_pacing(self) -> None:
+        args = _gallery_base_args()
+        self.assertEqual(args[args.index("--retries") + 1], "0")
+        self.assertEqual(args[args.index("--sleep-extractor") + 1], "5")
+        self.assertEqual(args[args.index("--sleep-request") + 1], "5")
+
+    def test_merge_gallery_files_concatenates_arrays(self) -> None:
+        with TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            first = root / "first.json"
+            second = root / "second.json"
+            output = root / "gallery.json"
+            first.write_text('[{"id": "100"}]', encoding="utf-8")
+            second.write_text('[{"id": "200"}]', encoding="utf-8")
+            _merge_gallery_files([first, second], output)
+            self.assertEqual(
+                json.loads(output.read_text(encoding="utf-8")),
+                [{"id": "100"}, {"id": "200"}],
+            )
+
+    def test_merge_gallery_files_rejects_non_array(self) -> None:
+        with TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            capture = root / "capture.json"
+            capture.write_text('{"id": "100"}', encoding="utf-8")
+            with self.assertRaisesRegex(SystemExit, "JSON array"):
+                _merge_gallery_files([capture], root / "gallery.json")
+
+    def test_validate_capture_ids_rejects_missing_requested_id(self) -> None:
+        with TemporaryDirectory() as tmp:
+            path = Path(tmp) / "thread_data.json"
+            path.write_text(
+                json.dumps({
+                    "root_post_id": "100",
+                    "source_url": "https://x.com/i/status/100",
+                    "posts": [
+                        {"post_id": "100", "metrics": {}},
+                        {"post_id": "200", "metrics": {}},
+                    ],
+                }),
+                encoding="utf-8",
+            )
+            _validate_capture_ids(path, ("100", "200"))
+            with self.assertRaisesRegex(SystemExit, "300"):
+                _validate_capture_ids(path, ("100", "300"))
+
     def test_migrate_defaults_to_dry_run(self) -> None:
         args = build_parser().parse_args(["migrate-media", "--id", "100"])
         self.assertEqual(args.cmd, "migrate-media")
@@ -29,4 +105,4 @@ class TwMediaCommandTests(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()

scripts/twitter/tw.py

@@ -27,6 +27,7 @@ if str(_HERE) not in sys.path:
     sys.path.insert(0, str(_HERE))
 
 from paths import COOKIES, DUMPS, FROZEN, HERE, SCRATCH, VAULT
+from models import load_thread
 
 try:
     from frozen import frozen_match, load_frozen_ids, require_writable
@@ -178,46 +179,79 @@ def cmd_graph(args: argparse.Namespace) -> int:
     return 0
 
 
+def _capture_ids(post_id: str, branch_ids: list[str]) -> tuple[str, ...]:
+    """Return the primary id plus distinct explicit branch tips."""
+    return tuple(dict.fromkeys((post_id, *branch_ids)))
+
+
+def _gallery_base_args() -> list[str]:
+    """Return the shared safe, paced gallery-dl arguments."""
+    return [
+        "gallery-dl",
+        "--cookies",
+        str(COOKIES),
+        "--retries",
+        "0",
+        "--sleep-extractor",
+        "5",
+        "--sleep-request",
+        "5",
+        "-o",
+        "text-tweets=true",
+        "-o",
+        "conversations=true",
+    ]
+
+
+def _merge_gallery_files(paths: list[Path], output: Path) -> None:
+    """Concatenate gallery-dl JSON arrays for local ingest."""
+    merged: list[object] = []
+    for path in paths:
+        data: object = json.loads(path.read_text(encoding="utf-8"))
+        if not isinstance(data, list):
+            raise SystemExit(f"gallery-dl output is not a JSON array: {path}")
+        merged.extend(data)
+    output.write_text(
+        json.dumps(merged, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+        newline="\n",
+    )
+
+
+def _validate_capture_ids(path: Path, capture_ids: tuple[str, ...]) -> None:
+    """Fail if gallery-dl omitted an explicitly requested tip."""
+    thread = load_thread(path)
+    present = {post.post_id for post in thread.posts}
+    missing = [post_id for post_id in capture_ids if post_id not in present]
+    if missing:
+        raise SystemExit(f"requested tips missing from capture: {', '.join(missing)}")
+
+
 def cmd_refetch(args: argparse.Namespace) -> int:
-    """gallery-dl a thread into the scratch dir, then run ``ingest_gallery.py`` to materialize thread_data.json."""
+    """Capture one spine tip plus explicit branch tips into one dump."""
     post_id = args.id
-    check_frozen(post_id)
+    capture_ids = _capture_ids(post_id, args.branch)
+    for capture_id in capture_ids:
+        check_frozen(capture_id)
     if not COOKIES.is_file():
         raise SystemExit(f"missing {COOKIES}")
+
     out = scratch_dir(post_id)
-    media = out / "media"
-    media.mkdir(parents=True, exist_ok=True)
-    url = f"https://x.com/i/status/{post_id}"
+    out.mkdir(parents=True, exist_ok=True)
+    capture_paths: list[Path] = []
+    for capture_id in capture_ids:
+        capture = out / f"gallery_{capture_id}.json"
+        url = f"https://x.com/i/status/{capture_id}"
+        with capture.open("w", encoding="utf-8") as fh:
+            subprocess.check_call(
+                [*_gallery_base_args(), "--dump-json", url],
+                stdout=fh,
+            )
+        capture_paths.append(capture)
+
     gallery = out / "gallery.json"
-    with gallery.open("w", encoding="utf-8") as fh:
-        subprocess.check_call(
-            [
-                "gallery-dl",
-                "--cookies",
-                str(COOKIES),
-                "--dump-json",
-                "-o",
-                "text-tweets=true",
-                "-o",
-                "conversations=true",
-                url,
-            ],
-            stdout=fh,
-        )
-    run(
-        [
-            "gallery-dl",
-            "--cookies",
-            str(COOKIES),
-            "-D",
-            str(media),
-            "-o",
-            "text-tweets=true",
-            "-o",
-            "conversations=true",
-            url,
-        ]
-    )
+    _merge_gallery_files(capture_paths, gallery)
+    source_url = f"https://x.com/i/status/{post_id}"
     run(
         [
             sys.executable,
@@ -227,11 +261,19 @@ def cmd_refetch(args: argparse.Namespace) -> int:
             "--out",
             str(out / "thread_data.json"),
             "--source-url",
-            url,
+            source_url,
             "--root",
             post_id,
         ]
     )
+    _validate_capture_ids(out / "thread_data.json", capture_ids)
+
+    media = out / "media"
+    media.mkdir(parents=True, exist_ok=True)
+    for capture_id in capture_ids:
+        url = f"https://x.com/i/status/{capture_id}"
+        run([*_gallery_base_args(), "-D", str(media), url])
+
     print(f"refetch -> {out}")
     return 0
 
@@ -562,6 +604,14 @@ def build_parser() -> argparse.ArgumentParser:
     ):
         p = sub.add_parser(name)
         p.add_argument("--id", required=True)
+        if name in ("refetch", "refresh"):
+            p.add_argument(
+                "--branch",
+                action="append",
+                default=[],
+                metavar="TIP_ID",
+                help="capture and merge this explicit branch tip; repeatable",
+            )
         if name in ("emit", "refresh"):
             p.add_argument("--tip", action="store_true", help="use --id as tip")
             p.add_argument("--slug", default=None)