Scripts update: new media convention.
This commit is contained in:
ed committed
1 parent
2945eca13c
commit
24a82aa46c
16 files changed
+1897
-443
No files matched your search
+60
-110
@@ -1,83 +1,32 @@
|
||||
"""Select which still a note cites, and optionally attach OCR under it.
|
||||
"""Attach OCR transcripts under the cited origin media. Local variants are
|
||||
not publication locations; visual selection is driven by the canonical
|
||||
manifest's `embed` flag.
|
||||
|
||||
Notes currently say:
|
||||
|
||||
Media (not lifted): `1736…_GBgHT…_orig.png`
|
||||
|
||||
After lift they will say:
|
||||
|
||||
|
||||
|
||||
This script only rewrites those two shapes. It does not upload.
|
||||
|
||||
python media_embed.py --thread <assets-thread-dir> --notes <archive-thread-dir> --media-id GBgHT_2WIAAKuNq --show crt
|
||||
python media_embed.py --thread <assets-thread-dir> --notes <archive-thread-dir> --media-id GBgHT_2WIAAKuNq --attach-ocr
|
||||
python media_embed.py --thread <assets-thread-dir> --notes <archive-thread-dir> --media-id <id> --attach-ocr
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
IMAGE_MD = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
|
||||
LIFTED_LINE = re.compile(r"^Media \(not lifted\): (.+)$", re.M)
|
||||
try:
|
||||
from .media_manifest import atomic_write_json, find_location, selected_url, validate_manifest
|
||||
except ImportError: # pragma: no cover - script-mode import
|
||||
if str(Path(__file__).resolve().parent) not in sys.path:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from media_manifest import atomic_write_json, find_location, selected_url, validate_manifest
|
||||
|
||||
|
||||
def load_media(path: Path) -> dict:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def save_media(path: Path, data: dict) -> None:
|
||||
path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def group_items(items: list[dict]) -> dict[tuple[str, str], dict[str, dict]]:
|
||||
groups: dict[tuple[str, str], dict[str, dict]] = {}
|
||||
for item in items:
|
||||
key = (str(item.get("post_id") or ""), str(item.get("media_id") or ""))
|
||||
groups.setdefault(key, {})[str(item.get("role") or "")] = item
|
||||
return groups
|
||||
|
||||
|
||||
def set_embed(items: list[dict], media_id: str, show: str) -> None:
|
||||
found = False
|
||||
for item in items:
|
||||
if str(item.get("media_id") or "") != media_id:
|
||||
continue
|
||||
if item.get("role") in ("orig", "crt", "crt_outline"):
|
||||
item["embed"] = item.get("role") == show
|
||||
found = True
|
||||
if not found:
|
||||
raise SystemExit(f"no orig/crt row for media_id {media_id}")
|
||||
|
||||
|
||||
def token_for(item: dict) -> str:
|
||||
url = item.get("url")
|
||||
if url:
|
||||
return str(url)
|
||||
return str(item.get("filename") or "")
|
||||
|
||||
|
||||
def rewrite_notes(notes_dir: Path, old: str, new: str) -> int:
|
||||
if not old or old == new:
|
||||
return 0
|
||||
changed = 0
|
||||
for path in notes_dir.glob("*.md"):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if old not in text:
|
||||
continue
|
||||
path.write_text(text.replace(old, new), encoding="utf-8")
|
||||
changed += 1
|
||||
print(f"rewrote {path.name}")
|
||||
return changed
|
||||
OCR_DETAILS = "<summary>Text from still</summary>"
|
||||
|
||||
|
||||
def ocr_block(text: str) -> str:
|
||||
body = text.strip("\n")
|
||||
return (
|
||||
"<details>\n"
|
||||
"<summary>Text from still</summary>\n"
|
||||
f"{OCR_DETAILS}\n"
|
||||
"\n"
|
||||
"```\n"
|
||||
f"{body}\n"
|
||||
@@ -94,23 +43,20 @@ def attach_ocr(notes_dir: Path, image_token: str, ocr_text: str) -> int:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if image_token not in text:
|
||||
continue
|
||||
if "<summary>Text from still</summary>" in text and image_token in text:
|
||||
if OCR_DETAILS in text and image_token in text:
|
||||
print(f"ocr already near token in {path.name}")
|
||||
continue
|
||||
# After the line that cites the still.
|
||||
lines = text.splitlines(keepends=True)
|
||||
out: list[str] = []
|
||||
i = 0
|
||||
inserted = False
|
||||
while i < len(lines):
|
||||
out.append(lines[i])
|
||||
if image_token in lines[i] and not inserted:
|
||||
if not lines[i].endswith("\n"):
|
||||
out[-1] = lines[i] + "\n"
|
||||
for i, line in enumerate(lines):
|
||||
out.append(line)
|
||||
if image_token in line and not inserted:
|
||||
if not line.endswith("\n"):
|
||||
out[-1] = line + "\n"
|
||||
out.append("\n")
|
||||
out.append(block + "\n")
|
||||
inserted = True
|
||||
i += 1
|
||||
if inserted:
|
||||
path.write_text("".join(out), encoding="utf-8")
|
||||
changed += 1
|
||||
@@ -123,50 +69,54 @@ def main(argv: list[str] | None = None) -> int:
|
||||
parser.add_argument("--thread", type=Path, required=True, help="assets/threads/.../ folder")
|
||||
parser.add_argument("--notes", type=Path, required=True, help="archive/threads/.../ folder")
|
||||
parser.add_argument("--media-id", required=True)
|
||||
parser.add_argument("--show", choices=("orig", "crt"), default=None)
|
||||
parser.add_argument("--attach-ocr", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
media_path = args.thread / "media.json"
|
||||
data = load_media(media_path)
|
||||
items = data["items"]
|
||||
groups = group_items(items)
|
||||
match = None
|
||||
pair = None
|
||||
for key, roles in groups.items():
|
||||
if key[1] == args.media_id:
|
||||
match = key
|
||||
pair = roles
|
||||
break
|
||||
if pair is None or match is None:
|
||||
raise SystemExit(f"media_id {args.media_id} not in {media_path}")
|
||||
|
||||
if args.show:
|
||||
if args.show not in pair:
|
||||
raise SystemExit(f"no role={args.show} for {args.media_id}")
|
||||
old_item = next((pair[r] for r in ("crt", "orig") if pair.get(r, {}).get("embed")), None)
|
||||
if old_item is None:
|
||||
old_item = pair.get("orig") or pair.get("crt")
|
||||
set_embed(items, args.media_id, args.show)
|
||||
save_media(media_path, data)
|
||||
new_item = pair[args.show]
|
||||
rewrite_notes(args.notes, token_for(old_item), token_for(new_item))
|
||||
print(f"embed {args.media_id} -> {args.show} ({token_for(new_item)})")
|
||||
raise SystemExit(
|
||||
"local variants are not publishable locations; after confirming an "
|
||||
"origin failure, use tw.py fallback with the desired --role"
|
||||
)
|
||||
if not args.attach_ocr:
|
||||
parser.print_help()
|
||||
return 2
|
||||
|
||||
if args.attach_ocr:
|
||||
ocr = pair.get("ocr")
|
||||
if ocr is None:
|
||||
raise SystemExit(f"no role=ocr for {args.media_id}; run ocr_pass.py first")
|
||||
ocr_path = args.thread / str(ocr["filename"])
|
||||
if not ocr_path.is_file():
|
||||
raise SystemExit(f"missing {ocr_path}")
|
||||
shown = next((pair[r] for r in ("crt", "orig") if pair.get(r, {}).get("embed")), None)
|
||||
if shown is None:
|
||||
shown = pair.get("orig") or pair.get("crt")
|
||||
attach_ocr(args.notes, token_for(shown), ocr_path.read_text(encoding="utf-8"))
|
||||
media_path = args.thread / "media.json"
|
||||
data = json.loads(media_path.read_text(encoding="utf-8"))
|
||||
issues = validate_manifest(data)
|
||||
if issues:
|
||||
raise SystemExit(f"manifest invalid: {issues[0]}")
|
||||
|
||||
target = None
|
||||
for item in data.get("items") or []:
|
||||
if (
|
||||
str(item.get("media_id") or "") == args.media_id
|
||||
and str(item.get("role") or "") == "ocr"
|
||||
):
|
||||
target = item
|
||||
break
|
||||
if target is None:
|
||||
raise SystemExit(f"no role=ocr for {args.media_id}; run ocr_pass.py first")
|
||||
ocr_path = args.thread / str(target.get("filename") or "")
|
||||
if not ocr_path.is_file():
|
||||
raise SystemExit(f"missing {ocr_path}")
|
||||
|
||||
visible = None
|
||||
for item in data.get("items") or []:
|
||||
if str(item.get("media_id") or "") != args.media_id:
|
||||
continue
|
||||
if item.get("embed") and selected_url(item):
|
||||
visible = item
|
||||
break
|
||||
if visible is None:
|
||||
raise SystemExit("media group has no embedded origin")
|
||||
image_token = str(selected_url(visible) or "")
|
||||
if not image_token:
|
||||
raise SystemExit("media group has no selected HTTPS location")
|
||||
attach_ocr(args.notes, image_token, ocr_path.read_text(encoding="utf-8"))
|
||||
atomic_write_json(media_path, data)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user