Scripts update: new media convention.
This commit is contained in:
1 parent
2945eca13c
commit
24a82aa46c
16 files changed
+1897
-443
No files matched your search
@@ -0,0 +1,315 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
try:
|
||||
from .frozen import frozen_match
|
||||
from .media_manifest import (
|
||||
SCHEMA_VERSION,
|
||||
atomic_write_json,
|
||||
find_location,
|
||||
hash_file,
|
||||
new_derived_item,
|
||||
new_fallback_location,
|
||||
new_original_item,
|
||||
validate_manifest,
|
||||
)
|
||||
from .media_refs import apply_rewrite_plan, atomic_write_text, plan_thread_rewrites
|
||||
except ImportError: # pragma: no cover - script-mode import
|
||||
if str(Path(__file__).resolve().parent) not in sys.path:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from frozen import frozen_match
|
||||
from media_manifest import (
|
||||
SCHEMA_VERSION,
|
||||
atomic_write_json,
|
||||
find_location,
|
||||
hash_file,
|
||||
new_derived_item,
|
||||
new_fallback_location,
|
||||
new_original_item,
|
||||
validate_manifest,
|
||||
)
|
||||
from media_refs import apply_rewrite_plan, atomic_write_text, plan_thread_rewrites
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MigrationResult:
|
||||
state: str
|
||||
changed: bool
|
||||
issues: tuple[str, ...]
|
||||
item_count: int
|
||||
|
||||
|
||||
def media_id_from_url(url: str, index: int) -> str:
|
||||
segment = Path(unquote(urlparse(url).path)).name
|
||||
return segment if segment else f"m{index}"
|
||||
|
||||
|
||||
def source_urls(thread_data: dict[str, Any]) -> dict[tuple[str, str], str]:
|
||||
result: dict[tuple[str, str], str] = {}
|
||||
for post in thread_data.get("posts") or []:
|
||||
post_id = str(post.get("post_id") or "")
|
||||
for index, url in enumerate(post.get("media_urls") or [], start=1):
|
||||
key = (post_id, media_id_from_url(str(url), index))
|
||||
if key in result and result[key] != url:
|
||||
raise ValueError(f"duplicate source mapping for {key!r}")
|
||||
result[key] = str(url)
|
||||
return result
|
||||
|
||||
|
||||
def reference_maps(
|
||||
legacy: dict[str, Any], canonical: dict[str, Any]
|
||||
) -> tuple[dict[str, str], dict[str, list[str]]]:
|
||||
canonical_by_key = {
|
||||
(
|
||||
str(item.get("post_id") or ""),
|
||||
str(item.get("media_id") or ""),
|
||||
str(item.get("role") or ""),
|
||||
): item
|
||||
for item in canonical.get("items") or []
|
||||
}
|
||||
filename_origins: dict[str, str] = {}
|
||||
fallback_origins: dict[str, list[str]] = {}
|
||||
for old in legacy.get("items") or []:
|
||||
if old.get("role") != "orig":
|
||||
continue
|
||||
key = (
|
||||
str(old.get("post_id") or ""),
|
||||
str(old.get("media_id") or ""),
|
||||
"orig",
|
||||
)
|
||||
item = canonical_by_key[key]
|
||||
origin = next(
|
||||
location
|
||||
for location in item["locations"]
|
||||
if location.get("kind") == "origin"
|
||||
)["url"]
|
||||
filename_origins[str(old.get("filename") or "")] = origin
|
||||
old_url = str(old.get("url") or "")
|
||||
if old_url.startswith("https://"):
|
||||
fallback_origins.setdefault(old_url, []).append(origin)
|
||||
return filename_origins, fallback_origins
|
||||
|
||||
|
||||
def corpus_inventory(
|
||||
assets_root: Path,
|
||||
note_root: Path,
|
||||
frozen_ids: set[str],
|
||||
*,
|
||||
now: str,
|
||||
) -> dict[str, object]:
|
||||
media_paths = sorted(assets_root.rglob("media.json"))
|
||||
counts: dict[str, int] = {
|
||||
"input_items": 0,
|
||||
"migratable": 0,
|
||||
"frozen_skipped": 0,
|
||||
"missing_origin_mapping": 0,
|
||||
"missing_local_file": 0,
|
||||
"ambiguous_note_reference": 0,
|
||||
"invalid_legacy_row": 0,
|
||||
}
|
||||
threads: list[dict[str, object]] = []
|
||||
frozen_hashes: dict[str, str] = {}
|
||||
for media_path in media_paths:
|
||||
asset_dir = media_path.parent
|
||||
relative = asset_dir.relative_to(assets_root).as_posix()
|
||||
match = frozen_match(asset_dir, frozen_ids)
|
||||
legacy = json.loads(media_path.read_text(encoding="utf-8"))
|
||||
is_legacy = legacy.get("schema_version") != SCHEMA_VERSION
|
||||
legacy_items = list(legacy.get("items") or [])
|
||||
unique_legacy = {
|
||||
(
|
||||
str(item.get("post_id") or ""),
|
||||
str(item.get("media_id") or ""),
|
||||
str(item.get("role") or ""),
|
||||
)
|
||||
for item in legacy_items
|
||||
}
|
||||
counts["input_items"] += len(unique_legacy)
|
||||
for path in sorted(asset_dir.iterdir()):
|
||||
if path.is_file() and path.name != "media.json":
|
||||
key = f"{relative}/{path.name}"
|
||||
frozen_hashes[key] = hash_file(path)
|
||||
for archive in sorted((note_root / relative).glob("*.md")) if (note_root / relative).is_dir() else []:
|
||||
key = f"{relative}/{archive.name}"
|
||||
frozen_hashes[key] = hash_file(archive)
|
||||
if match is not None:
|
||||
counts["frozen_skipped"] += len(unique_legacy)
|
||||
threads.append({
|
||||
"relative": relative,
|
||||
"state": "frozen_skipped",
|
||||
"frozen_id": match,
|
||||
"item_count": len(unique_legacy),
|
||||
})
|
||||
continue
|
||||
thread_data_path = asset_dir / "thread_data.json"
|
||||
if not thread_data_path.is_file():
|
||||
counts["missing_origin_mapping"] += len(unique_legacy)
|
||||
threads.append({
|
||||
"relative": relative,
|
||||
"state": "missing_origin_mapping",
|
||||
"item_count": len(unique_legacy),
|
||||
"issues": ["missing thread_data.json"],
|
||||
})
|
||||
continue
|
||||
thread_data = json.loads(thread_data_path.read_text(encoding="utf-8"))
|
||||
try:
|
||||
canonical, issues = canonical_from_legacy(legacy, thread_data, asset_dir, now)
|
||||
except Exception as exc:
|
||||
counts["invalid_legacy_row"] += len(unique_legacy)
|
||||
threads.append({
|
||||
"relative": relative,
|
||||
"state": "invalid_legacy_row",
|
||||
"item_count": len(unique_legacy),
|
||||
"issues": [str(exc)],
|
||||
})
|
||||
continue
|
||||
if issues:
|
||||
counts["invalid_legacy_row"] += len(unique_legacy)
|
||||
threads.append({
|
||||
"relative": relative,
|
||||
"state": "invalid_legacy_row",
|
||||
"item_count": len(unique_legacy),
|
||||
"issues": list(issues),
|
||||
})
|
||||
continue
|
||||
counts["migratable"] += len(canonical["items"])
|
||||
thread_entry: dict[str, object] = {
|
||||
"relative": relative,
|
||||
"state": "migratable" if is_legacy else "current",
|
||||
"item_count": len(canonical["items"]),
|
||||
}
|
||||
threads.append(thread_entry)
|
||||
expected_total = (
|
||||
counts["migratable"]
|
||||
+ counts["frozen_skipped"]
|
||||
+ counts["missing_origin_mapping"]
|
||||
+ counts["missing_local_file"]
|
||||
+ counts["ambiguous_note_reference"]
|
||||
+ counts["invalid_legacy_row"]
|
||||
)
|
||||
if counts["input_items"] != expected_total:
|
||||
raise SystemExit("corpus accounting mismatch")
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"generated_at": now,
|
||||
"counts": counts,
|
||||
"threads": threads,
|
||||
"frozen_hashes": frozen_hashes,
|
||||
}
|
||||
|
||||
|
||||
def canonical_from_legacy(
|
||||
legacy: dict[str, Any],
|
||||
thread_data: dict[str, Any],
|
||||
asset_dir: Path,
|
||||
now: str,
|
||||
) -> tuple[dict[str, Any], tuple[str, ...]]:
|
||||
sources = source_urls(thread_data)
|
||||
items: list[dict[str, Any]] = []
|
||||
issues: list[str] = []
|
||||
seen: set[tuple[str, str, str]] = set()
|
||||
for old in legacy.get("items") or []:
|
||||
role = str(old.get("role") or "")
|
||||
post_id = str(old.get("post_id") or "")
|
||||
media_id = str(old.get("media_id") or "")
|
||||
filename = str(old.get("filename") or "")
|
||||
key = (post_id, media_id, role)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
if role == "orig":
|
||||
origin_url = sources.get((post_id, media_id))
|
||||
if origin_url is None:
|
||||
issues.append(f"missing origin mapping: {post_id}/{media_id}")
|
||||
continue
|
||||
item = new_original_item(
|
||||
post_id=post_id,
|
||||
media_id=media_id,
|
||||
handle=str(old.get("handle") or ""),
|
||||
origin_url=origin_url,
|
||||
filename=filename,
|
||||
local_path=asset_dir / filename,
|
||||
now=now,
|
||||
)
|
||||
old_url = str(old.get("url") or "")
|
||||
if old_url.startswith("https://"):
|
||||
local = find_location(item, "local")
|
||||
item["locations"].append(
|
||||
new_fallback_location(
|
||||
provider="catbox",
|
||||
url=old_url,
|
||||
content_hash=str(local.get("sha256") or "") or None,
|
||||
recorded_at=now,
|
||||
uploaded_at=None,
|
||||
)
|
||||
)
|
||||
items.append(item)
|
||||
elif role in {"crt", "crt_outline", "denoise", "ocr"}:
|
||||
item = new_derived_item(legacy=old, asset_dir=asset_dir, now=now)
|
||||
old_url = str(old.get("url") or "")
|
||||
if old_url.startswith("https://"):
|
||||
local = find_location(item, "local")
|
||||
item["locations"].append(
|
||||
new_fallback_location(
|
||||
provider="catbox",
|
||||
url=old_url,
|
||||
content_hash=str(local.get("sha256") or "") or None,
|
||||
recorded_at=now,
|
||||
uploaded_at=None,
|
||||
)
|
||||
)
|
||||
items.append(item)
|
||||
else:
|
||||
issues.append(f"invalid legacy role: {post_id}/{media_id}/{role}")
|
||||
manifest = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"root_post_id": str(legacy.get("root_post_id") or ""),
|
||||
"items": items,
|
||||
"mirrors": copy.deepcopy(legacy.get("mirrors") or []),
|
||||
}
|
||||
issues.extend(validate_manifest(manifest))
|
||||
return manifest, tuple(issues)
|
||||
|
||||
|
||||
def migrate_legacy_thread(
|
||||
asset_dir: Path,
|
||||
note_dir: Path,
|
||||
*,
|
||||
now: str,
|
||||
apply: bool,
|
||||
) -> MigrationResult:
|
||||
media_path = asset_dir / "media.json"
|
||||
legacy = json.loads(media_path.read_text(encoding="utf-8"))
|
||||
if legacy.get("schema_version") == SCHEMA_VERSION:
|
||||
return MigrationResult("current", False, (), len(legacy.get("items") or []))
|
||||
thread_data = json.loads((asset_dir / "thread_data.json").read_text(encoding="utf-8"))
|
||||
canonical, issues = canonical_from_legacy(legacy, thread_data, asset_dir, now)
|
||||
if issues:
|
||||
return MigrationResult("blocked", False, issues, len(legacy.get("items") or []))
|
||||
if apply:
|
||||
filename_origins, fallback_origins = reference_maps(legacy, canonical)
|
||||
plan = plan_thread_rewrites(
|
||||
note_dir,
|
||||
filename_origins=filename_origins,
|
||||
fallback_origins=fallback_origins,
|
||||
)
|
||||
if plan.issues:
|
||||
return MigrationResult("blocked", False, plan.issues, len(legacy.get("items") or []))
|
||||
note_backup: dict[Path, str] = {}
|
||||
try:
|
||||
for rewrite in plan.files:
|
||||
note_backup[rewrite.path] = rewrite.before
|
||||
atomic_write_json(media_path, canonical)
|
||||
apply_rewrite_plan(plan)
|
||||
except Exception:
|
||||
for path, before in note_backup.items():
|
||||
atomic_write_text(path, before)
|
||||
raise
|
||||
return MigrationResult("migratable", apply, (), len(legacy.get("items") or []))
|
||||
Reference in new issue
Block a user