Scripts update: new media convention.
This commit is contained in:
ed committed
1 parent
2945eca13c
commit
24a82aa46c
16 files changed
+1897
-443
No files matched your search
@@ -1,204 +1,19 @@
|
||||
"""Upload curated stills to catbox and rewrite archive notes to HTTPS.
|
||||
"""Deprecated.
|
||||
|
||||
Reads userhash from vault secrets/credentials.toml [catbox]. Never prints it.
|
||||
|
||||
python lift_catbox.py --thread ASSETS_THREAD --notes ARCHIVE_THREAD --orig
|
||||
|
||||
--orig marks every on-disk orig row publish=true then uploads those.
|
||||
Default without --orig: only rows already publish=true.
|
||||
|
||||
After upload, any note token that is the local filename (backticked or
|
||||
bare) or a previous url for that row becomes .
|
||||
|
||||
Catbox rate-limits the userhash per minute. Algorithm:
|
||||
|
||||
- One attempt per upload. No curl retries. A failure is a failure —
|
||||
the user retries the whole lift later (the script persists progress
|
||||
after each success, so partial work survives).
|
||||
- 60-second sleep between consecutive uploads in the same run.
|
||||
- 200 KB/s upload cap per curl.
|
||||
- On empty/non-URL body: sleep 120s before raising, so a manual
|
||||
retry minutes later doesn't arrive during catbox's cooldown.
|
||||
- media.json is rewritten after each successful upload (atomic),
|
||||
not at the end of the run.
|
||||
- Errors are sanitized to redact the userhash before printing.
|
||||
Bulk lift has been retired. Use `tw.py fallback --id <id> --media-id <media-id> --role <role>
|
||||
--confirm-origin-unavailable` after confirming the origin is unavailable.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
from paths import VAULT as VAULT_ROOT
|
||||
|
||||
API = "https://catbox.moe/user/api.php"
|
||||
MAX_TIME = 180
|
||||
THROTTLE_SECS = 60.0
|
||||
LIMIT_RATE = "200k"
|
||||
EMPTY_BACKOFF = 120.0
|
||||
|
||||
|
||||
def _redact_cmd(cmd: list[str]) -> str:
|
||||
out: list[str] = []
|
||||
for tok in cmd:
|
||||
if tok.startswith("userhash="):
|
||||
out.append("userhash=<redacted>")
|
||||
else:
|
||||
out.append(tok)
|
||||
return " ".join(out)
|
||||
|
||||
|
||||
def load_userhash(vault: Path) -> str:
|
||||
path = vault / "secrets" / "credentials.toml"
|
||||
if not path.is_file():
|
||||
raise SystemExit(f"missing {path}")
|
||||
text = path.read_text(encoding="utf-8")
|
||||
try:
|
||||
data = tomllib.loads(text)
|
||||
h = str((data.get("catbox") or {}).get("userhash") or "").strip()
|
||||
except tomllib.TOMLDecodeError:
|
||||
h = ""
|
||||
in_catbox = False
|
||||
for raw in text.splitlines():
|
||||
line = raw.strip()
|
||||
if line.startswith("[") and line.endswith("]"):
|
||||
in_catbox = line[1:-1].strip().lower() == "catbox"
|
||||
continue
|
||||
if in_catbox and line.lower().startswith("userhash"):
|
||||
_, _, rest = line.partition("=")
|
||||
h = rest.strip().strip("\"'")
|
||||
break
|
||||
if not h:
|
||||
raise SystemExit("credentials.toml [catbox] userhash is empty")
|
||||
return h
|
||||
|
||||
|
||||
def upload(path: Path, userhash: str) -> str:
|
||||
cmd = [
|
||||
"curl.exe",
|
||||
"-sS",
|
||||
"--max-time",
|
||||
str(MAX_TIME),
|
||||
"--limit-rate",
|
||||
LIMIT_RATE,
|
||||
"-F",
|
||||
"reqtype=fileupload",
|
||||
"-F",
|
||||
f"userhash={userhash}",
|
||||
"-F",
|
||||
f"fileToUpload=@{path}",
|
||||
API,
|
||||
]
|
||||
try:
|
||||
raw = subprocess.check_output(cmd, text=True, stderr=subprocess.STDOUT)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
time.sleep(EMPTY_BACKOFF)
|
||||
raise SystemExit(
|
||||
f"catbox upload failed for {path.name} "
|
||||
f"(curl exit {exc.returncode}); waited {EMPTY_BACKOFF:.0f}s before exit. "
|
||||
f"cmd: {_redact_cmd(cmd)}"
|
||||
) from None
|
||||
url = raw.strip()
|
||||
if not url.startswith("https://"):
|
||||
time.sleep(EMPTY_BACKOFF)
|
||||
raise SystemExit(
|
||||
f"catbox refused {path.name} (empty/non-URL body); "
|
||||
f"waited {EMPTY_BACKOFF:.0f}s before exit. body[:200]={url[:200]!r}"
|
||||
)
|
||||
return url
|
||||
|
||||
|
||||
def rewrite_notes(notes_dir: Path, old_tokens: list[str], url: str) -> int:
|
||||
embed = f""
|
||||
changed = 0
|
||||
for path in notes_dir.glob("*.md"):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
new = text
|
||||
for token in old_tokens:
|
||||
if not token:
|
||||
continue
|
||||
new = new.replace(f"`{token}`", embed)
|
||||
new = new.replace(f"", embed)
|
||||
new = re.sub(r"^Media \(not lifted\):\s*", "", new, flags=re.M)
|
||||
while True:
|
||||
split = re.sub(
|
||||
r"(!\[[^\]]*\]\([^)]+\))\s+(!\[[^\]]*\]\([^)]+\))",
|
||||
r"\1\n\n\2",
|
||||
new,
|
||||
)
|
||||
if split == new:
|
||||
break
|
||||
new = split
|
||||
if new != text:
|
||||
path.write_text(new, encoding="utf-8", newline="\n")
|
||||
changed += 1
|
||||
print(f"rewrote {path.name}")
|
||||
return changed
|
||||
|
||||
|
||||
def persist(media_path: Path, data: dict) -> None:
|
||||
media_path.write_text(
|
||||
json.dumps(data, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
def main(argv: list[str] | None = None) -> int: # pragma: no cover - deprecated entry
|
||||
del argv
|
||||
sys.stderr.write(
|
||||
"lift_catbox.py is retired; use tw.py fallback with --confirm-origin-unavailable\n"
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--vault", type=Path, default=VAULT_ROOT)
|
||||
parser.add_argument("--thread", type=Path, required=True)
|
||||
parser.add_argument("--notes", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--orig",
|
||||
action="store_true",
|
||||
help="publish+upload every orig file that exists on disk",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
media_path = args.thread / "media.json"
|
||||
data = json.loads(media_path.read_text(encoding="utf-8"))
|
||||
items = data.setdefault("items", [])
|
||||
if args.orig:
|
||||
for item in items:
|
||||
if item.get("role") == "orig":
|
||||
item["publish"] = True
|
||||
if "embed" not in item:
|
||||
item["embed"] = True
|
||||
|
||||
userhash = None if args.dry_run else load_userhash(args.vault)
|
||||
uploaded = 0
|
||||
publishable = [it for it in items if it.get("publish")]
|
||||
for i, item in enumerate(publishable):
|
||||
if i > 0 and not args.dry_run:
|
||||
print(f"throttling {THROTTLE_SECS:.0f}s before next upload…")
|
||||
time.sleep(THROTTLE_SECS)
|
||||
name = str(item.get("filename") or "")
|
||||
src = args.thread / name
|
||||
if not src.is_file():
|
||||
print(f"skip missing {name}")
|
||||
continue
|
||||
old = [name, str(item.get("url") or "")]
|
||||
if args.dry_run:
|
||||
print(f"would upload {name}")
|
||||
continue
|
||||
assert userhash is not None
|
||||
url = upload(src, userhash)
|
||||
item["url"] = url
|
||||
uploaded += 1
|
||||
print(f"uploaded {name} -> {url}")
|
||||
rewrite_notes(args.notes, old, url)
|
||||
persist(media_path, data)
|
||||
print(f"persisted media.json ({uploaded} done so far)")
|
||||
|
||||
print(f"uploaded {uploaded}")
|
||||
return 0
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in new issue
Block a user