From 1ceb4b1583774f1940f7d114d262a5174aa68592 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Tue, 25 Aug 2026 14:34:46 -0400 Subject: [PATCH] checkpoint: cleanup and review. --- README.md | 22 ++-- assets/README.md | 6 +- meta/conventions.md | 8 +- notes/How threads are organized.md | 6 +- scripts/twitter/README.md | 96 +++++++++------- scripts/twitter/backup_assets.py | 9 +- scripts/twitter/catbox_client.py | 88 +++++++++++---- scripts/twitter/tests/test_catbox_client.py | 119 ++++++++++++++++++++ scripts/twitter/tw.py | 39 +++---- secrets/README.md | 22 +++- 10 files changed, 299 insertions(+), 116 deletions(-) create mode 100644 scripts/twitter/tests/test_catbox_client.py diff --git a/README.md b/README.md index 5199b94..ea3f1e7 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ Recovered threads and submerged records. -An [Obsidian](https://obsidian.md) vault published with [Quartz 5](https://quartz.jzhao.xyz) to GitHub Pages. The site is a convenient host for Twitter/X threads and other material that is not reliably web-indexed. +An [Obsidian](https://obsidian.md) vault published with [Quartz 5](https://quartz.jzhao.xyz) to GitHub Pages. The site keeps Twitter/X threads and other material that search does not keep. -Live (after Pages is enabled): [threadwell.edsabode.dev](https://threadwell.edsabode.dev) +Live: [threadwell.edsabode.dev](https://threadwell.edsabode.dev) ## Post an issue for an inaccuracy @@ -12,7 +12,7 @@ I'll correct it. ## Layout -Open **this repo root** as the Obsidian vault. Quartz is only the publisher; it lives in `site/`. +Open **this repo root** as the Obsidian vault. Quartz lives in `site/` and only publishes. ``` archive/ # public — primary sources @@ -20,7 +20,7 @@ archive/ # public — primary sources # one thread: index.md is the author's chain sources/ # talks, pages, other records canvases/ # public — .canvas maps -notes/ # public — authored / distilled writing +notes/ # public — authored writing templates/ # not published meta/ # not published — conventions assets/ # not published, gitignored — working media @@ -33,7 +33,7 @@ site/ # local Quartz clone (gitignored; own remotes) .github/workflows/deploy.yml ``` -`archive/`, `canvases/`, and `notes/` are public by default. `site/` is a separate git checkout of Quartz (`origin` → `jackyzha0/quartz`). Threadwell's only remote is `Ed94/Threadwell`. +`archive/`, `canvases/`, and `notes/` publish by default. `site/` is a separate checkout of Quartz (`origin` → `jackyzha0/quartz`). This repo's remote is `Ed94/Threadwell`. ## Frontmatter @@ -51,19 +51,19 @@ site/ # local Quartz clone (gitignored; own remotes) | `draft` | `true` omits the page from the site | | `tags`, `description`, `aliases` | as usual | -`draft: true` only hides a page from the **site**. A public GitHub repo still contains the file. Use `private/` for anything that must not be pushed. +`draft: true` hides a page from the **site**. A public GitHub repo still contains the file. Use `private/` for anything that must not be pushed. Templates: `templates/archive-source.md`, `templates/authored-note.md` (Templater placeholders). ## Images and embeds -Images: absolute HTTPS URLs from the original provider when possible. Notes do not embed local binaries. +Notes cite the original host URL. ```md ![](https://pbs.twimg.com/media/FhAidkhUoAA3FJz?format=png&name=orig) ``` -Keep local copies in `assets/` (not committed). Do not point notes at `assets/` or `./media/`. +Keep local copies in `assets/` (not committed). Notes do not point at `assets/` or `./media/`. YouTube and tweets (Quartz Obsidian-flavored Markdown): @@ -78,7 +78,7 @@ Odysee (iframe; copy the embed URL from Odysee share): ``` -Remote URLs and iframes are left as-is in the static build. +Remote URLs and iframes stay as written in the static build. ## Obsidian plugins @@ -121,7 +121,7 @@ Open http://localhost:8080 ## Add content and republish 1. New thread: create `archive/threads///`. Templater → `archive-source` → save as `index.md` (the author's chain). Off-spine replies get their own notes in that folder. See `notes/How threads are organized.md`. -2. Transcribe posts in order. Upload images to the image host; paste HTTPS URLs. Optional YouTube/Odysee/tweet embeds. +2. Transcribe posts in order. Paste the original HTTPS image URL. Optional YouTube/Odysee/tweet embeds. 3. Distill in `notes/` with `authored-note`. Wikilink the archive record. 4. Set `draft: false` and `status: published` when the capture has been reviewed. 5. Preview: `cd site; npx quartz build -d .. --serve` @@ -138,4 +138,4 @@ Engine license: MIT (Quartz). Vault content: Unlicense (`LICENSE`). ## Notice -I'm not a web dev. Look away at the scripts this is just done because it doesn't exist, so it was done badly. +I'm not a web dev. Look away at the scripts. This exists because it didn't, so it was done badly. diff --git a/assets/README.md b/assets/README.md index 39f7637..ba9851b 100644 --- a/assets/README.md +++ b/assets/README.md @@ -1 +1,5 @@ -Working media for Threadwell. Mirrors archive/threads/// plus handles//. Gitignored except this file. Not published. Notes never link here; published media cites the original provider's HTTPS URL. A separate `tw.py backup --id ` command mirrors these assets to configured storage; alternative-host uploads are a manual fallback used only after the origin is confirmed unavailable. Do not link notes at this folder. +Working media. Same tree as `archive/threads///`, plus `handles//` for avatars. + +Gitignored except this file. Not on the site. Notes never link here. + +Published images use the original host URL. `tw.py backup` copies this tree to the configured store. `tw.py fallback` uploads one file after the origin is gone. diff --git a/meta/conventions.md b/meta/conventions.md index 6ae6dce..113b23c 100644 --- a/meta/conventions.md +++ b/meta/conventions.md @@ -25,7 +25,7 @@ draft: true | `in_reply_to` | string | Foreign parent of a spine, if any | | `parent_post_id` | string | Branch notes only: post this branch replies to | -`draft: true` hides the page from the built site. It does **not** hide the file from a public GitHub repo. Truly private material goes in `private/` (gitignored). +`draft: true` hides the page from the built site. A public GitHub repo still contains the file. Private material goes in `private/` (gitignored). # Folders @@ -37,7 +37,7 @@ draft: true | `notes/` | yes | Authored writing | | `templates/` | no | Templater / core templates | | `meta/` | no | Vault rules, local attachments | -| `assets/` | no, gitignored | Working media before upload | +| `assets/` | no, gitignored | Working media | | `secrets/` | no, gitignored | Cookies and image-host credentials | | `scripts/` | no | Tracked tooling. Quartz-ignored | | `private/` | no, gitignored | Never publish, never push | @@ -48,10 +48,10 @@ A thread folder holds `index.md` (spine) and one markdown file per off-spine rep # Images -Always absolute HTTPS URLs from the original provider when possible (`pbs.twimg.com`, `video.twimg.com`). The vault also retains a local, unpublished copy under `assets/` using the same relative taxonomy as the archive. A separately requested backup may mirror those assets to configured storage; alternative-host uploads are a manual fallback used only after the origin is confirmed unavailable, and the original URL and every fallback URL remain recorded in `media.json`. Do not commit local binaries for published notes. Working files sit in `assets/` (gitignored), not in `meta/attachments/` except unrelated Obsidian drops. +Use the original host URL (`pbs.twimg.com`, `video.twimg.com`). A local copy stays under `assets/` with the same relative path as the archive. `tw.py backup` may copy that tree to the configured store. `tw.py fallback` uploads one file after the origin is gone. `media.json` keeps every URL. Do not commit local binaries for published notes. ```md -![](https://files.catbox.moe/example.png) +![](https://pbs.twimg.com/media/EXAMPLE?format=png&name=orig) ``` # Video diff --git a/notes/How threads are organized.md b/notes/How threads are organized.md index 07399fc..b48dbd8 100644 --- a/notes/How threads are organized.md +++ b/notes/How threads are organized.md @@ -32,15 +32,15 @@ The spine links out under the post that grew the branch. Each branch note links ## Cross-author conversations -When a thread alternates between authors (A replies to B replies to A), the archive directory is owned by the OP — the post with no parent in the chain — and the spine walks every post in chronological order regardless of handle. Cross-author posts appear inline in the spine. Responder handles do not get their own directories; the conversation lives in one folder under the OP's name. +When a thread alternates between authors, the folder belongs to the opening post. The spine walks every post in time order. Replies stay in that folder. The other authors do not get a second copy. ## Tags -Every imported thread starts with `archive`, `twitter`, and the author's handle. Topic tags (`vulkan`, `crt`, `alsa`, …) are added by hand when the thread is reviewed. +Every thread starts with `archive`, `twitter`, and the author's handle. Topic tags (`vulkan`, `crt`, `alsa`, …) are added by hand when the thread is reviewed. ## Images -Pictures on the site are direct HTTPS URLs from the original image host when possible. Working copies of files live outside what GitHub Pages serves. A note that is still being prepared is marked `draft: true` and does not appear on the site. +Pictures on the site are the original host URLs. Working copies live outside what GitHub Pages serves. A note still being prepared is marked `draft: true` and stays off the site. ## Reading diff --git a/scripts/twitter/README.md b/scripts/twitter/README.md index e04ef49..9d7fe98 100644 --- a/scripts/twitter/README.md +++ b/scripts/twitter/README.md @@ -1,36 +1,43 @@ # twitter -Front door: `tw.py`. Individual scripts still work. - -From the vault root: +Front door: `tw.py`. Run from the vault root. ``` -python C:\projects\Threadwell\scripts\twitter\tw.py graph --id -python C:\projects\Threadwell\scripts\twitter\tw.py refresh --id --tip -python C:\projects\Threadwell\scripts\twitter\tw.py refresh --id --tip --branch -python C:\projects\Threadwell\scripts\twitter\tw.py add-branch --id --from -python C:\projects\Threadwell\scripts\twitter\tw.py emit --id --tip --preserve-existing -python C:\projects\Threadwell\scripts\twitter\tw.py audit-media --id -python C:\projects\Threadwell\scripts\twitter\tw.py publish --id -python C:\projects\Threadwell\scripts\twitter\tw.py sync --handle -python C:\projects\Threadwell\scripts\twitter\tw.py backup --id -python C:\projects\Threadwell\scripts\twitter\tw.py fallback --id --media-id --role --confirm-origin-unavailable -python C:\projects\Threadwell\scripts\twitter\tw.py restore-origin --id --media-id -python C:\projects\Threadwell\scripts\twitter\tw.py migrate-media --id -python C:\projects\Threadwell\scripts\twitter\tw.py migrate-media --all --apply -python C:\projects\Threadwell\scripts\twitter\tw.py reslug --all -python C:\projects\Threadwell\scripts\twitter\tw.py reslug --all --apply +python scripts/twitter/tw.py graph --id +python scripts/twitter/tw.py refresh --id --tip +python scripts/twitter/tw.py refresh --id --tip --branch +python scripts/twitter/tw.py add-branch --id --from +python scripts/twitter/tw.py emit --id --tip --preserve-existing +python scripts/twitter/tw.py audit-media --id +python scripts/twitter/tw.py publish --id +python scripts/twitter/tw.py sync --handle +python scripts/twitter/tw.py backup --id +python scripts/twitter/tw.py fallback --id --media-id --role --confirm-origin-unavailable +python scripts/twitter/tw.py restore-origin --id --media-id +python scripts/twitter/tw.py migrate-media --id +python scripts/twitter/tw.py reslug --all +python scripts/twitter/tw.py reslug --all --apply ``` -`paths.py` sets vault = two parents above this file, dumps = `../manual_slop/docs/twitter`, scratch = `../Threadwell-ai/scratch`. +`paths.py` finds the vault two parents above this file. Dumps live at `../manual_slop/docs/twitter`. Scratch lives at `../Threadwell-ai/scratch`. -`refresh` = refetch (cookies) + ingest + emit `--force` + media_merge. `--tip` treats `--id` as the tip and walks back to the OP regardless of handle. Repeatable `--branch ` captures only those additional explicit tip paths and merges them into the same `thread_data.json`; it does not discover replies automatically. Captures run sequentially with gallery-dl retries disabled and five-second extractor/request pacing. Each thread emits exactly one archive directory owned by the OP (the post with `reply_to_id == None`); cross-author responders do not get a directory. New notes default to `draft: false` (publish by default). Does not commit. Frozen ids abort. +## Capture -`emit --preserve-existing` is the safe completion step after inspecting a fresh scratch capture. Fresh records win for ids the provider returned; posts present only in the existing emitted `thread_data.json` are retained after the command verifies both captures share a conversation root. This prevents an incomplete gallery-dl refresh from deleting archived replies. The merged scratch dataset keeps the requested `--id` as its stored spine tip before normal emission and media-manifest reconciliation. +`refresh` is refetch + ingest + emit `--force` + media merge. `--tip` treats `--id` as the tip and walks back to the opening post. Repeat `--branch ` to capture extra tips into the same `thread_data.json`. It does not invent replies. Captures run one at a time. gallery-dl retries are off. Extractor and request sleeps are five seconds. -`add-branch` incrementally extends an already emitted thread. `--id` must be the original stored spine tip; repeatable `--from ` values are each captured once. The command locally retains each node's missing attachment path and entire visible descendant subtree, merges only previously absent posts into the existing `thread_data.json`, downloads media for the new posts, and re-emits with the original spine unchanged. It preserves existing origin/local/fallback/derived/OCR media records and adds provider-origin plus local-copy records for new branch media. A visible leaf is only the deepest reply returned by that capture; the command does not query descendants individually or claim provider-wide completeness. Gallery-dl may return only the ancestor chain when replies are behind nested show-more cursors; then the command adds nothing and known reply/stream-tip ids must be supplied explicitly. Fallback upload remains a separate explicit action after origin unavailability is confirmed. +Each thread writes one archive folder. The owner is the opening post (`reply_to_id` is empty). Other authors stay in that folder. New notes set `draft: false`. The command does not commit. Frozen ids abort. -New emits publish by default. `tw.py publish --id ` is only used to flip a thread that was captured with `draft: true` (rare after this change). The usual review path is: +For a thread already on disk, stage with `refetch`, compare ids, then: + +``` +python scripts/twitter/tw.py emit --id --tip --preserve-existing +``` + +Fresh records win for ids the provider returned. Posts only in the existing `thread_data.json` stay if both captures share a conversation root. That stops an incomplete gallery-dl run from deleting archived replies. + +`add-branch` extends an emitted thread. `--id` is the stored spine tip. Each `--from ` is captured once. The command keeps that node's missing attachment path and the visible descendant subtree, merges new posts, downloads their media, and re-emits with the spine unchanged. Existing media rows stay. New media get origin and local-copy rows. A visible leaf is only the deepest reply that capture returned. + +Usual path: 1. `tw.py graph --id …` — confirm tip vs root. 2. `tw.py refresh --id … --tip` if the dump stored a tip as root. @@ -38,42 +45,45 @@ New emits publish by default. `tw.py publish --id ` is only used to f Site wikilinks must be `[[archive/threads//]]`. Short `[[slug]]` 404s. -## Media policy +## Media -Published Twitter/X media cites the original provider's HTTPS URL by default. The vault also retains a local, unpublished copy under `assets/threads///` using the same relative taxonomy as the archive. A separately requested backup may mirror those assets to configured storage. Alternative-host uploads are a manual fallback used only after the origin is confirmed unavailable; the original URL and every fallback URL remain recorded in `media.json`. +Published images cite the original host URL. A local copy stays under `assets/threads///`. `backup` copies that tree to the configured store. `fallback` uploads one file after the origin is gone. `media.json` keeps the origin URL and every fallback URL. -The `lift` command is retired. Use `fallback` for a single media item after explicit confirmation, or `restore-origin` to select the immutable provider URL again. The Catbox userhash and backup destination id/root are read from `secrets/credentials.toml` by the script. Never paste them into chat. +`lift` is retired. Use `fallback` for one item, or `restore-origin` to select the original URL again. -Frozen threads (ids listed in `do_not_refetch.txt`, matched against every captured post id) are read-only across refetch, refresh, emit, manifest migration, note rewrite, fallback, publication selection, and tracked backup operations. Read-only `audit-media` may still inspect them. +Frozen threads (any captured post id listed in `do_not_refetch.txt`) are read-only for refetch, refresh, emit, migration, rewrite, fallback, and backup. `audit-media` may still inspect them. -## Canonical thread directories +## Thread directories -Each thread archive directory is named from the rendered title that appears in `index.md` (`-`). `tw.py reslug --all` reads each existing `index.md` frontmatter date and title, derives the expected directory, and reports every mapping without writing. `tw.py reslug --all --apply` preflights every pair, refuses the entire apply on any non-frozen conflict, and moves assets first with the archive directory immediately after; if the archive move fails, assets roll back. If a destination is occupied, the apply aborts without `-2`, `-3`, or any other suffix. Frozen ids in `do_not_refetch.txt` are reported as `frozen` and skipped. +A thread folder is named from the title in `index.md`: `-`. `reslug --all` reports every mapping. `reslug --all --apply` preflights, refuses the whole apply on any non-frozen conflict, and moves assets first. If the archive move fails, assets roll back. An occupied destination aborts. Frozen ids are reported and skipped. -Future emits use the same rendered title for the directory and `index.md` frontmatter, so normal emit does this automatically — no flag is needed on emit, refresh, or add-branch. Hand-move only the archive or only the asset is not supported; both must move as a pair. Each old archive prefix is replaced exactly once in mutable `*.md` and `*.canvas` outside `.git/`, `site/`, `assets/`, `secrets/`, `node_modules/`, and frozen archive dirs, so mappings do not cascade. No aliases, redirect pages, compatibility folders, suffix fallback, or network contact. +Later emits use that same title for the folder and the frontmatter. Move both sides together. Each old archive prefix is replaced once in mutable `*.md` and `*.canvas` outside `.git/`, `site/`, `assets/`, `secrets/`, `node_modules/`, and frozen archive dirs. + +## Secrets + +`secrets/twitter_cookies.txt` and `secrets/credentials.toml` stay on the machine that runs the capture. The scripts read them. They never print them. The Catbox hash goes in an HTTPS form body, not on a process command line. The backup command prints `synced` or `error`, not the destination path. ## Pieces | Script | Job | |---|---| -| `tw.py` | locate / graph / refetch / add-branch / emit / refresh / ocr / merge / sync / publish / migrate-media / backup / fallback / restore-origin | +| `tw.py` | locate / graph / refetch / add-branch / emit / refresh / ocr / merge / sync / publish / migrate-media / backup / fallback / restore-origin / reslug | | `graph_dry_run.py` | tip graph, no vault writes | | `ingest_gallery.py` | gallery-dl JSON → `thread_data.json` | -| `emit_archive.py` | JSON → notes + assets. Uses canonical media locations. `--force` preserves derived/fallback rows | -| `media_migrate.py` | legacy `media.json` → canonical locations, dry-run or apply | +| `emit_archive.py` | JSON → notes + assets. `--force` keeps derived and fallback rows | +| `media_migrate.py` | old `media.json` → location records, dry-run or apply | | `media_audit.py` | local integrity, reference counts, mirror freshness, optional origin check | | `media_manifest.py` | schema, validation, merge, selection, atomic I/O | | `media_refs.py` | media-aware markup and note rewrite planning | -| `backup_assets.py` | sparse, hash-verified copy of one thread asset dir | -| `catbox_client.py` | one sanitized Catbox upload; no batching or retry | -| `fallback_media.py` | confirm-origin fallback activation and restore-origin | +| `backup_assets.py` | hash-verified copy of one thread asset dir | +| `catbox_client.py` | one Catbox upload | +| `fallback_media.py` | fallback activation and restore-origin | | `frozen.py` | resolve frozen ids against every captured post id | -| `lift_catbox.py` | retired entry; points operators to `tw.py fallback` | +| `lift_catbox.py` | retired. Use `tw.py fallback` | | `ocr_pass.py` | Umi HTTP `:1224` / tesseract / Windows OCR | -| `media_embed.py` | `--attach-ocr` only; refuses `--show crt` | -| `media_merge.py` | restore derived rows under canonical API | -| `crt_pass.py` | stock slang only; ShaderGlass is the look reference | +| `media_embed.py` | `--attach-ocr` only | +| `media_merge.py` | restore derived rows | +| `crt_pass.py` | stock slang only. ShaderGlass is the look reference | | `do_not_refetch.txt` | frozen id list | -| `tests/` | isolated, deterministic unit tests | - -Secrets: `secrets/twitter_cookies.txt`, `secrets/credentials.toml` `[catbox] userhash` and `[backup] id`/`root`. The script reads them; never paste into chat, tests, or commits. +| `tests/` | isolated unit tests | +| `unfinished_bloat/` | leftover probe code. Not the front door | diff --git a/scripts/twitter/backup_assets.py b/scripts/twitter/backup_assets.py index 2ee7ee4..d792191 100644 --- a/scripts/twitter/backup_assets.py +++ b/scripts/twitter/backup_assets.py @@ -83,7 +83,6 @@ def backup_thread( ) -> BackupResult: """Mirror ``asset_dir`` into ``destination_root/`` and verify every payload by sha256.""" media_path = asset_dir / "media.json" - media_path = asset_dir / "media.json" raw = json.loads(media_path.read_text(encoding="utf-8")) manifest = _from_wire_dict(raw) relative = asset_dir.relative_to(assets_root) @@ -107,7 +106,7 @@ def backup_thread( ] try: if require_destination_root and not destination_root.is_dir(): - raise OSError(f"destination root unavailable: {destination_root}") + raise OSError("destination root unavailable") destination.mkdir(parents=True, exist_ok=True) for source in sorted(asset_dir.iterdir(), key=lambda path: path.name): if source.is_file(): @@ -122,7 +121,11 @@ def backup_thread( mirror["completed_at"] = now mirror["verified_at"] = now except OSError as exc: - mirror["error"] = str(exc)[:300] + text = str(exc) + if str(destination_root) in text or str(destination) in text: + mirror["error"] = "backup copy or verification failed" + else: + mirror["error"] = text[:300] raw["mirrors"] = others + [mirror] media_path.write_text( json.dumps(_manifest_to_wire_for_disk(manifest, raw), indent=2, ensure_ascii=False) + "\n", diff --git a/scripts/twitter/catbox_client.py b/scripts/twitter/catbox_client.py index 18844c3..91d59b7 100644 --- a/scripts/twitter/catbox_client.py +++ b/scripts/twitter/catbox_client.py @@ -1,13 +1,23 @@ -"""Catbox userhash loader and uploader (curl.exe wrapper).""" +"""Load the Catbox userhash and upload one file. + +The hash is read from ``secrets/credentials.toml``. It never appears on a +process command line or in printed output. +""" from __future__ import annotations -import subprocess +import ssl import tomllib from pathlib import Path +from secrets import token_hex +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +API: str = "https://catbox.moe/user/api.php" +TIMEOUT_SECS: int = 180 def load_userhash(vault: Path) -> str: - """Load the catbox ``userhash`` from ``vault/secrets/credentials.toml`` (TOML or lenient line scan).""" + """Return the Catbox ``userhash`` from ``vault/secrets/credentials.toml``.""" path = vault / "secrets" / "credentials.toml" if not path.is_file(): raise RuntimeError(f"missing {path}") @@ -32,30 +42,60 @@ def load_userhash(vault: Path) -> str: return value +def _encode_multipart( + fields: list[tuple[str, str]], + file_field: str, + file_path: Path, +) -> tuple[bytes, str]: + """Build a multipart body. Field values stay in the body, not in argv.""" + boundary = "----Threadwell" + token_hex(16) + chunks: list[bytes] = [] + for name, value in fields: + chunks.append( + ( + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="{name}"\r\n' + f"\r\n" + f"{value}\r\n" + ).encode("utf-8") + ) + filename = file_path.name.replace('"', "_") + chunks.append( + ( + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="{file_field}"; ' + f'filename="{filename}"\r\n' + f"Content-Type: application/octet-stream\r\n" + f"\r\n" + ).encode("utf-8") + ) + chunks.append(file_path.read_bytes()) + chunks.append(f"\r\n--{boundary}--\r\n".encode("utf-8")) + return b"".join(chunks), f"multipart/form-data; boundary={boundary}" + + def upload_file(path: Path, userhash: str) -> str: - """Upload ``path`` to catbox.moe via curl.exe and return the public HTTPS URL.""" - cmd = [ - "curl.exe", - "-sS", - "--max-time", - "180", - "--limit-rate", - "200k", - "-F", - "reqtype=fileupload", - "-F", - f"userhash={userhash}", - "-F", - f"fileToUpload=@{path}", - "https://catbox.moe/user/api.php", - ] + """POST ``path`` to catbox.moe and return the public HTTPS URL.""" + body, content_type = _encode_multipart( + [("reqtype", "fileupload"), ("userhash", userhash)], + "fileToUpload", + path, + ) + request = Request(API, data=body, method="POST") + request.add_header("Content-Type", content_type) try: - raw = subprocess.check_output(cmd, text=True, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as exc: + with urlopen( + request, + timeout=TIMEOUT_SECS, + context=ssl.create_default_context(), + ) as resp: + url = resp.read().decode("utf-8", errors="replace").strip() + except HTTPError as exc: raise RuntimeError( - f"catbox upload failed for {path.name} (curl exit {exc.returncode})" + f"catbox upload failed for {path.name} (HTTP {exc.code})" ) from None - url = raw.strip() + except URLError: + raise RuntimeError(f"catbox upload failed for {path.name}") from None if not url.startswith("https://"): raise RuntimeError(f"catbox refused {path.name}: non-URL response") - return url \ No newline at end of file + return url diff --git a/scripts/twitter/tests/test_catbox_client.py b/scripts/twitter/tests/test_catbox_client.py new file mode 100644 index 0000000..fc6ffc4 --- /dev/null +++ b/scripts/twitter/tests/test_catbox_client.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch +from urllib.error import HTTPError, URLError +from urllib.request import Request + +from twitter.catbox_client import load_userhash, upload_file + + +class _FakeResponse: + def __init__(self, body: bytes) -> None: + self._body = body + + def read(self) -> bytes: + return self._body + + def __enter__(self) -> _FakeResponse: + return self + + def __exit__(self, *_args: object) -> bool: + return False + + +class CatboxClientTests(unittest.TestCase): + def test_load_userhash_reads_toml(self) -> None: + with TemporaryDirectory() as raw: + vault = Path(raw) + (vault / "secrets").mkdir() + (vault / "secrets" / "credentials.toml").write_text( + '[catbox]\nuserhash = "test-hash"\n', + encoding="utf-8", + ) + self.assertEqual(load_userhash(vault), "test-hash") + + def test_load_userhash_rejects_empty(self) -> None: + with TemporaryDirectory() as raw: + vault = Path(raw) + (vault / "secrets").mkdir() + (vault / "secrets" / "credentials.toml").write_text( + "[catbox]\nuserhash = \"\"\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(RuntimeError, "userhash is empty"): + load_userhash(vault) + + def test_upload_posts_form_without_subprocess(self) -> None: + with TemporaryDirectory() as raw: + src = Path(raw) / "still.png" + src.write_bytes(b"png") + captured: dict[str, object] = {} + + def fake_urlopen( + request: Request, + timeout: int, + context: object, + ) -> _FakeResponse: + captured["data"] = request.data + captured["timeout"] = timeout + return _FakeResponse(b"https://files.catbox.moe/abc.png") + + with patch("twitter.catbox_client.urlopen", fake_urlopen): + url = upload_file(src, "test-hash") + self.assertEqual(url, "https://files.catbox.moe/abc.png") + body = captured["data"] + assert isinstance(body, bytes) + self.assertIn(b"test-hash", body) + self.assertIn(b"reqtype", body) + self.assertIn(b"png", body) + self.assertEqual(captured["timeout"], 180) + + def test_error_message_omits_userhash(self) -> None: + with TemporaryDirectory() as raw: + src = Path(raw) / "still.png" + src.write_bytes(b"png") + + def boom( + request: Request, + timeout: int, + context: object, + ) -> _FakeResponse: + del request, timeout, context + raise URLError("refused") + + with patch("twitter.catbox_client.urlopen", boom): + with self.assertRaises(RuntimeError) as ctx: + upload_file(src, "test-hash") + self.assertNotIn("test-hash", str(ctx.exception)) + + def test_http_error_omits_userhash(self) -> None: + with TemporaryDirectory() as raw: + src = Path(raw) / "still.png" + src.write_bytes(b"png") + + def boom( + request: Request, + timeout: int, + context: object, + ) -> _FakeResponse: + del timeout, context + raise HTTPError( + request.full_url, + 500, + "server", + hdrs=None, # type: ignore[arg-type] + fp=None, + ) + + with patch("twitter.catbox_client.urlopen", boom): + with self.assertRaises(RuntimeError) as ctx: + upload_file(src, "test-hash") + self.assertNotIn("test-hash", str(ctx.exception)) + self.assertIn("HTTP 500", str(ctx.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/twitter/tw.py b/scripts/twitter/tw.py index 5ec5813..b60f285 100644 --- a/scripts/twitter/tw.py +++ b/scripts/twitter/tw.py @@ -1,17 +1,16 @@ -"""Front door for common Twitter archive ops. +"""Front door for Twitter archive work. From the vault root: python scripts/twitter/tw.py graph --id 1692565070583136348 python scripts/twitter/tw.py refresh --id 1692565070583136348 --tip python scripts/twitter/tw.py add-branch --id 1692565070583136348 --from - python scripts/twitter/tw.py lift --id 1692565070583136348 --orig python scripts/twitter/tw.py ocr --id 1692565070583136348 python scripts/twitter/tw.py locate --id 1692565070583136348 -refresh = refetch + emit --force (--tip uses the --id as tip). -publish = draft: false on the thread index (the only switch). -Does not commit. Never prints cookies or userhash. +refresh = refetch + emit --force. --tip treats --id as the tip. +publish flips draft: false on an old capture. New notes already publish. +Does not commit. Never prints cookies, userhash, or the backup root. """ from __future__ import annotations @@ -79,7 +78,6 @@ def scratch_dir(post_id: str) -> Path: def locate(post_id: str) -> tuple[Path | None, Path | None]: """Return ``(asset_dir, note_dir)`` for the first media.json whose root or captured posts contain ``post_id``.""" assets_root = VAULT / "assets" / "threads" - assets_root = VAULT / "assets" / "threads" notes_root = VAULT / "archive" / "threads" if not assets_root.is_dir(): return None, None @@ -558,24 +556,11 @@ def cmd_emit(args: argparse.Namespace) -> int: def cmd_lift(args: argparse.Namespace) -> int: - """Invoke ``lift_catbox.py`` to swap fallback URLs into the notes for ``args.id`` (with ``--orig`` to restore).""" - post_id = args.id - orig = args.orig - assets, notes = locate(post_id) - if assets is None or notes is None: - raise SystemExit(f"no vault thread for {post_id} — emit first") - cmd = [ - sys.executable, - str(HERE / "lift_catbox.py"), - "--thread", - str(assets), - "--notes", - str(notes), - ] - if orig: - cmd.append("--orig") - run(cmd) - return 0 + """Retired. Point the operator at ``fallback``.""" + del args + raise SystemExit( + "lift is retired; use fallback --confirm-origin-unavailable" + ) def cmd_ocr(args: argparse.Namespace) -> int: @@ -731,7 +716,7 @@ def cmd_migrate_media(args: argparse.Namespace) -> int: if not args.apply: print("dry-run: explicit --apply required to write") return 0 - print("corpus migrate-media --apply not yet implemented; awaiting Task 11") + print("corpus-wide migrate-media --apply is not implemented") return 0 @@ -764,7 +749,9 @@ def cmd_backup(args: argparse.Namespace) -> int: now=_now_iso(), require_destination_root=True, ) - print(result) + print(result.state) + if result.error: + print(result.error) return 0 if result.state == "synced" else 2 diff --git a/secrets/README.md b/secrets/README.md index 0514f6c..a419035 100644 --- a/secrets/README.md +++ b/secrets/README.md @@ -1 +1,21 @@ -Put cookies.txt (Netscape, for gallery-dl) and catbox.env here. Gitignored except this file. Never commit secrets. Never paste them into chat. +Local credentials. Gitignored except this file. + +| File | Contents | +| --- | --- | +| `twitter_cookies.txt` | Netscape cookies for gallery-dl | +| `credentials.toml` | `[catbox] userhash`, `[backup] id` and `root` | + +Shape of `credentials.toml`: + +```toml +[catbox] +userhash = "" + +[backup] +id = "cozy" +root = "" +``` + +The scripts read these files. They never print the values. + +Do not commit this folder. Do not paste the contents into chat or issues.