From 5b087286cf3992a23e2cf1c1657f36a63edfbaee Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 23 Aug 2026 22:41:38 -0400 Subject: [PATCH] script update --- scripts/twitter/README.md | 81 +++------ scripts/twitter/emit_archive.py | 55 ++++++ scripts/twitter/tw.py | 295 ++++++++++++++++++++++++++++++++ 3 files changed, 377 insertions(+), 54 deletions(-) create mode 100644 scripts/twitter/tw.py diff --git a/scripts/twitter/README.md b/scripts/twitter/README.md index ab2c55b..d01d0e9 100644 --- a/scripts/twitter/README.md +++ b/scripts/twitter/README.md @@ -1,66 +1,39 @@ # twitter -Emit Threadwell archive notes from a `thread_data.json` dump. No fetch. Lift and CRT snapshot are still manual. - -## Emit +Front door: `tw.py`. Individual scripts still work. ``` -python C:\projects\Threadwell\scripts\twitter\emit_archive.py --input C:\projects\manual_slop\docs\twitter\2064858927829745887 --vault C:\projects\Threadwell --slug steamos-init --archived 2026-08-23 +python C:\projects\Threadwell\scripts\twitter\tw.py locate --id +python C:\projects\Threadwell\scripts\twitter\tw.py graph --id +python C:\projects\Threadwell\scripts\twitter\tw.py refresh --id --tip +python C:\projects\Threadwell\scripts\twitter\tw.py lift --id --orig +python C:\projects\Threadwell\scripts\twitter\tw.py ocr --id ``` -`--tip ` climbs same-handle `reply_to` and uses that chain as `index.md`. +`refresh` = refetch (cookies) + ingest + emit `--force` + media_merge. `--tip` treats `--id` as the tip (climb same-handle `reply_to`). Does not flip `draft`. Does not commit. Frozen Onat ids abort. -`--all` skips a dump whose `post_id` already exists unless `--force`. +Usual publish path after you like the note: -`--force` rewrites `media.json` to orig-only. Restore CRT/OCR rows with `media_merge.py`. +1. `tw.py graph --id …` — confirm tip vs root. +2. `tw.py refresh --id … --tip` if the dump stored a tip as root. +3. `tw.py lift --id … --orig` — catbox + rewrite `![](https://…)`. +4. You set `draft: false` and commit. -## Tip graph (no vault writes) +Site wikilinks must be `[[archive/threads//]]`. Short `[[slug]]` 404s. -``` -python C:\projects\Threadwell\scripts\twitter\graph_dry_run.py --input C:\projects\manual_slop\docs\twitter\ -``` +## Pieces -## CRT (stock slang only) +| Script | Job | +|---|---| +| `tw.py` | locate / graph / refetch / emit / refresh / lift / ocr / merge | +| `graph_dry_run.py` | tip graph, no vault writes | +| `ingest_gallery.py` | gallery-dl JSON → `thread_data.json` | +| `emit_archive.py` | JSON → notes + assets. `--force` now keeps CRT/OCR rows and prior catbox URLs | +| `lift_catbox.py` | upload `publish: true` (or `--orig`) | +| `ocr_pass.py` | Umi HTTP `:1224` / tesseract / Windows OCR | +| `media_embed.py` | `--show crt` / `--attach-ocr` | +| `media_merge.py` | restore extra rows if you still need it | +| `crt_pass.py` | stock slang only; ShaderGlass is the look reference | +| `do_not_refetch.txt` | Onat freeze list | -`crt_pass.py` nearest-scales, then runs **unpatched** `crt-lottes-multipass.slangp` / `crt-Cyclon.slangp` via `librashader-cli`. It does not edit shaders. ShaderGlass snapshot is still the look reference. - -``` -python C:\projects\Threadwell\scripts\twitter\crt_pass.py --in ORIG.png --out CRT.png --preset cyclon --nn 3 --shader-scale 100% --dry-run -``` - -`--nn 3` is Affinity 300%. `--shader-scale` is librashader `-d`. `--shaderglass` only launches the .sgp. - -If the plate is wrong, snapshot ShaderGlass yourself and drop `*_crt.png` next to the orig, then `media_merge.py`. - -## OCR - -``` -python C:\projects\Threadwell\scripts\twitter\ocr_pass.py --in ORIG.png --media-json THREAD\media.json -``` - -Prefers `umi-ocr` (`scoop install extras/umi-ocr`), then tesseract, then Windows.Media.Ocr. Writes `*_ocr.txt`. ShareX has no OCR CLI. - -## Select CRT vs orig in the note - -``` -python C:\projects\Threadwell\scripts\twitter\media_embed.py --thread ASSETS_THREAD --notes ARCHIVE_THREAD --media-id GBgHT_2WIAAKuNq --show crt -python C:\projects\Threadwell\scripts\twitter\media_embed.py --thread ASSETS_THREAD --notes ARCHIVE_THREAD --media-id GBgHT_2WIAAKuNq --attach-ocr -``` - -## Lift to catbox - -``` -python C:\projects\Threadwell\scripts\twitter\lift_catbox.py --thread ASSETS_THREAD --notes ARCHIVE_THREAD --orig -``` - -Reads `[catbox] userhash` from `secrets/credentials.toml`. Never prints it. Rewrites `Media (not lifted): \`file\`` to `![](https://files.catbox.moe/…)`. - -## Restore extra media.json rows - -``` -python C:\projects\Threadwell\scripts\twitter\media_merge.py --thread ASSETS_THREAD -``` - -## Frozen Onat - -`do_not_refetch.txt` — do not overwrite those three dumps. +Secrets: `secrets/twitter_cookies.txt`, `secrets/credentials.toml` `[catbox] userhash`. Never print them. diff --git a/scripts/twitter/emit_archive.py b/scripts/twitter/emit_archive.py index 5faccaf..5a72dad 100644 --- a/scripts/twitter/emit_archive.py +++ b/scripts/twitter/emit_archive.py @@ -108,6 +108,7 @@ def collect_media( "filename": filename, "publish": False, "url": None, + "embed": True, }) by_post.setdefault(post.post_id, []).append(filename) @@ -115,6 +116,50 @@ def collect_media( return items, media_by_post +def merge_existing_media( + dest_dir: Path, + items: list[dict[str, object]], + media_by_post: dict[str, tuple[str, ...]], +) -> tuple[list[dict[str, object]], dict[str, tuple[str, ...]]]: + """Keep CRT/OCR rows and prior lift URLs across --force emit.""" + old_path = dest_dir / "media.json" + if not old_path.is_file(): + return items, media_by_post + try: + old = json.loads(old_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return items, media_by_post + by_file = {str(i.get("filename") or ""): i for i in items} + for row in old.get("items") or []: + if not isinstance(row, dict): + continue + fn = str(row.get("filename") or "") + role = str(row.get("role") or "") + if role == "orig" and fn in by_file: + cur = by_file[fn] + if row.get("url"): + cur["url"] = row["url"] + if row.get("publish"): + cur["publish"] = row["publish"] + if "embed" in row: + cur["embed"] = row["embed"] + continue + if role in ("", "orig"): + continue + if fn in by_file: + continue + if fn and (dest_dir / fn).is_file(): + items.append(row) + by_post: dict[str, list[str]] = {} + for item in items: + if item.get("role") == "orig" and item.get("embed", True): + by_post.setdefault(str(item.get("post_id") or ""), []).append( + str(item.get("filename") or "") + ) + media_by_post = {pid: tuple(names) for pid, names in by_post.items()} + return items, media_by_post + + def missing_parent_ids(thread: ThreadData) -> list[str]: ids = by_id(thread) found: list[str] = [] @@ -381,6 +426,15 @@ def emit( if reuse_dir is None and force: _existing_ids, spines = collect_existing_ids(vault) reuse_dir = spines.get(thread.root_post_id) + if reuse_dir is None: + for post in thread.posts: + reuse_dir = spines.get(post.post_id) + if reuse_dir is not None: + break + if reuse_dir is None and slug: + candidate = vault / "archive" / "threads" / handle / slug + if candidate.is_dir(): + reuse_dir = candidate if reuse_dir is not None: dir_name = reuse_dir.name @@ -412,6 +466,7 @@ def emit( asset_dir.mkdir(parents=True, exist_ok=True) items, media_by_post = collect_media(thread, input_dir, asset_dir) + items, media_by_post = merge_existing_media(asset_dir, items, media_by_post) branch_links = { rid: wiki_branch(handle, dir_name, name) diff --git a/scripts/twitter/tw.py b/scripts/twitter/tw.py new file mode 100644 index 0000000..da8f4cd --- /dev/null +++ b/scripts/twitter/tw.py @@ -0,0 +1,295 @@ +"""Front door for common Twitter archive ops. + + python C:\\projects\\Threadwell\\scripts\\twitter\\tw.py graph --id 1692565070583136348 + python C:\\projects\\Threadwell\\scripts\\twitter\\tw.py refresh --id 1692565070583136348 --tip + python C:\\projects\\Threadwell\\scripts\\twitter\\tw.py lift --id 1692565070583136348 --orig + python C:\\projects\\Threadwell\\scripts\\twitter\\tw.py ocr --id 1692565070583136348 + python C:\\projects\\Threadwell\\scripts\\twitter\\tw.py locate --id 1692565070583136348 + +refresh = refetch + emit --force (--tip uses the --id as tip). +Does not commit. Does not flip draft. Never prints cookies or userhash. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +VAULT = Path(r"C:\projects\Threadwell") +DUMPS = Path(r"C:\projects\manual_slop\docs\twitter") +SCRATCH = Path(r"C:\projects\Threadwell-ai\scratch") +COOKIES = VAULT / "secrets" / "twitter_cookies.txt" +FROZEN = HERE / "do_not_refetch.txt" + + +def frozen_ids() -> set[str]: + out: set[str] = set() + if not FROZEN.is_file(): + return out + for line in FROZEN.read_text(encoding="utf-8").splitlines(): + s = line.strip() + if s and not s.startswith("#"): + out.add(s) + return out + + +def check_frozen(post_id: str) -> None: + if post_id in frozen_ids(): + raise SystemExit(f"frozen Onat dump {post_id} — will not refetch/overwrite") + + +def dump_dir(post_id: str) -> Path: + return DUMPS / post_id + + +def scratch_dir(post_id: str) -> Path: + return SCRATCH / f"refetch_{post_id}" + + +def locate(post_id: str) -> tuple[Path | None, Path | None]: + assets_root = VAULT / "assets" / "threads" + notes_root = VAULT / "archive" / "threads" + if not assets_root.is_dir(): + return None, None + for media in assets_root.rglob("media.json"): + try: + data = json.loads(media.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + hit = data.get("root_post_id") == post_id + if not hit: + for item in data.get("items") or []: + if str(item.get("post_id") or "") == post_id: + hit = True + break + if not hit: + td = media.parent / "thread_data.json" + if td.is_file(): + try: + t = json.loads(td.read_text(encoding="utf-8")) + except json.JSONDecodeError: + t = {} + if t.get("root_post_id") == post_id: + hit = True + else: + hit = any( + p.get("post_id") == post_id for p in (t.get("posts") or []) + ) + if hit: + rel = media.parent.relative_to(assets_root) + notes = notes_root / rel + return media.parent, notes if notes.is_dir() else None + return None, None + + +def run(cmd: list[str]) -> None: + print(">", " ".join(cmd)) + subprocess.check_call(cmd) + + +def cmd_locate(post_id: str) -> int: + dump = dump_dir(post_id) + scratch = scratch_dir(post_id) + assets, notes = locate(post_id) + print(f"dump {dump} exists={dump.is_dir()}") + print(f"scratch {scratch} exists={scratch.is_dir()}") + print(f"assets {assets}") + print(f"notes {notes}") + return 0 + + +def cmd_graph(post_id: str) -> int: + src = scratch_dir(post_id) + if not (src / "thread_data.json").is_file(): + src = dump_dir(post_id) + if not (src / "thread_data.json").is_file(): + raise SystemExit(f"no thread_data.json for {post_id}") + run([sys.executable, str(HERE / "graph_dry_run.py"), "--input", str(src)]) + return 0 + + +def cmd_refetch(post_id: str) -> int: + check_frozen(post_id) + if not COOKIES.is_file(): + raise SystemExit(f"missing {COOKIES}") + out = scratch_dir(post_id) + media = out / "media" + media.mkdir(parents=True, exist_ok=True) + url = f"https://x.com/i/status/{post_id}" + gallery = out / "gallery.json" + with gallery.open("w", encoding="utf-8") as fh: + subprocess.check_call( + [ + "gallery-dl", + "--cookies", + str(COOKIES), + "--dump-json", + "-o", + "text-tweets=true", + "-o", + "conversations=true", + url, + ], + stdout=fh, + ) + run( + [ + "gallery-dl", + "--cookies", + str(COOKIES), + "-D", + str(media), + "-o", + "text-tweets=true", + "-o", + "conversations=true", + url, + ] + ) + run( + [ + sys.executable, + str(HERE / "ingest_gallery.py"), + "--json", + str(gallery), + "--out", + str(out / "thread_data.json"), + "--source-url", + url, + "--root", + post_id, + ] + ) + print(f"refetch -> {out}") + return 0 + + +def cmd_emit(post_id: str, tip: bool, slug: str | None) -> int: + src = scratch_dir(post_id) + if not (src / "thread_data.json").is_file(): + src = dump_dir(post_id) + if not (src / "thread_data.json").is_file(): + raise SystemExit(f"no thread_data.json for {post_id}") + cmd = [ + sys.executable, + str(HERE / "emit_archive.py"), + "--input", + str(src), + "--vault", + str(VAULT), + "--force", + ] + if tip: + cmd.extend(["--tip", post_id]) + if slug: + cmd.extend(["--slug", slug]) + run(cmd) + assets, _notes = locate(post_id) + if assets is not None: + run([sys.executable, str(HERE / "media_merge.py"), "--thread", str(assets)]) + return 0 + + +def cmd_lift(post_id: str, orig: bool) -> int: + assets, notes = locate(post_id) + if assets is None or notes is None: + raise SystemExit(f"no vault thread for {post_id} — emit first") + cmd = [ + sys.executable, + str(HERE / "lift_catbox.py"), + "--thread", + str(assets), + "--notes", + str(notes), + ] + if orig: + cmd.append("--orig") + run(cmd) + return 0 + + +def cmd_ocr(post_id: str, engine: str) -> int: + assets, _notes = locate(post_id) + if assets is None: + raise SystemExit(f"no assets thread for {post_id}") + media_json = assets / "media.json" + n = 0 + for png in sorted(assets.glob("*_orig.png")) + sorted(assets.glob("*_orig.jpg")): + dest = png.with_name(png.name.replace("_orig.png", "_ocr.txt").replace("_orig.jpg", "_ocr.txt")) + run( + [ + sys.executable, + str(HERE / "ocr_pass.py"), + "--in", + str(png), + "--out", + str(dest), + "--engine", + engine, + "--media-json", + str(media_json), + ] + ) + n += 1 + print(f"ocr files {n}") + return 0 + + +def cmd_merge(post_id: str) -> int: + assets, _notes = locate(post_id) + if assets is None: + raise SystemExit(f"no assets thread for {post_id}") + run([sys.executable, str(HERE / "media_merge.py"), "--thread", str(assets)]) + return 0 + + +def cmd_refresh(post_id: str, tip: bool, slug: str | None) -> int: + cmd_refetch(post_id) + cmd_graph(post_id) + cmd_emit(post_id, tip=tip, slug=slug) + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="cmd", required=True) + for name in ("locate", "graph", "refetch", "emit", "lift", "ocr", "merge", "refresh"): + p = sub.add_parser(name) + p.add_argument("--id", required=True) + if name in ("emit", "refresh"): + p.add_argument("--tip", action="store_true", help="use --id as tip") + p.add_argument("--slug", default=None) + if name == "lift": + p.add_argument("--orig", action="store_true") + if name == "ocr": + p.add_argument( + "--engine", + default="umi", + choices=("auto", "umi", "tesseract", "windows"), + ) + args = parser.parse_args(argv) + post_id = args.id + if args.cmd == "locate": + return cmd_locate(post_id) + if args.cmd == "graph": + return cmd_graph(post_id) + if args.cmd == "refetch": + return cmd_refetch(post_id) + if args.cmd == "emit": + return cmd_emit(post_id, tip=args.tip, slug=args.slug) + if args.cmd == "lift": + return cmd_lift(post_id, orig=args.orig) + if args.cmd == "ocr": + return cmd_ocr(post_id, engine=args.engine) + if args.cmd == "merge": + return cmd_merge(post_id) + if args.cmd == "refresh": + return cmd_refresh(post_id, tip=args.tip, slug=args.slug) + raise SystemExit(f"unknown {args.cmd}") + + +if __name__ == "__main__": + raise SystemExit(main())