Make the scripts lesss bad python (still python...)
This commit is contained in:
1 parent
3805685d8f
commit
b64ab023ea
30 files changed
+1979
-653
No files matched your search
@@ -1,3 +1,4 @@
|
||||
"""Activate, restore, and dedupe fallback media locations for a thread."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
@@ -8,6 +9,8 @@ from typing import Any, Callable
|
||||
|
||||
try:
|
||||
from .media_manifest import (
|
||||
_from_wire_dict,
|
||||
_item_to_wire,
|
||||
atomic_write_json,
|
||||
find_location,
|
||||
hash_file,
|
||||
@@ -15,10 +18,19 @@ try:
|
||||
selected_url,
|
||||
)
|
||||
from .media_refs import atomic_write_text, remote_markup
|
||||
from .models import (
|
||||
MediaItem,
|
||||
MediaLocation,
|
||||
MediaLocationCheck,
|
||||
MediaManifest,
|
||||
Publication,
|
||||
)
|
||||
except ImportError: # pragma: no cover - script-mode import
|
||||
if str(Path(__file__).resolve().parent) not in sys.path:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from media_manifest import (
|
||||
_from_wire_dict,
|
||||
_item_to_wire,
|
||||
atomic_write_json,
|
||||
find_location,
|
||||
hash_file,
|
||||
@@ -26,15 +38,24 @@ except ImportError: # pragma: no cover - script-mode import
|
||||
selected_url,
|
||||
)
|
||||
from media_refs import atomic_write_text, remote_markup
|
||||
from models import (
|
||||
MediaItem,
|
||||
MediaLocation,
|
||||
MediaLocationCheck,
|
||||
MediaManifest,
|
||||
Publication,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FallbackResult:
|
||||
"""Outcome of activating one fallback location for a media item."""
|
||||
url: str
|
||||
reused: bool
|
||||
|
||||
|
||||
def replace_selected_url(note_dir: Path, old_url: str, new_url: str) -> None:
|
||||
"""Swap ``old_url`` for ``new_url`` in exactly one note under ``note_dir`` (no-op if already swapped)."""
|
||||
if old_url == new_url:
|
||||
return
|
||||
old_markup = remote_markup(old_url)
|
||||
@@ -56,27 +77,27 @@ def replace_selected_url(note_dir: Path, old_url: str, new_url: str) -> None:
|
||||
atomic_write_text(path, text.replace(old_markup, new_markup, 1))
|
||||
|
||||
|
||||
def _group(manifest: dict[str, Any], media_id: str) -> list[dict[str, Any]]:
|
||||
return [
|
||||
item
|
||||
for item in manifest.get("items") or []
|
||||
if str(item.get("media_id") or "") == media_id
|
||||
]
|
||||
def _group(manifest: MediaManifest, media_id: str) -> list[MediaItem]:
|
||||
"""Return the slice of ``manifest.items`` sharing a media_id (orig + derived variants)."""
|
||||
return [item for item in manifest.items if item.media_id == media_id]
|
||||
|
||||
|
||||
def _one(items: list[dict[str, Any]], role: str) -> dict[str, Any]:
|
||||
matches = [item for item in items if item.get("role") == role]
|
||||
def _one(items: list[MediaItem], role: str) -> MediaItem:
|
||||
"""Return the single item in ``items`` whose ``kind`` matches ``role`` (raises otherwise)."""
|
||||
matches = [item for item in items if item.kind == role]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"expected one role={role}, found {len(matches)}")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def _verified_local(item: dict[str, Any], asset_dir: Path) -> tuple[Path, str]:
|
||||
def _verified_local(item: MediaItem, asset_dir: Path) -> tuple[Path, str]:
|
||||
"""Return ``(local_path, sha256)`` of ``item``'s local location, verified against its declared hash."""
|
||||
local = find_location(item, "local")
|
||||
if local is None or local.get("integrity") != "present":
|
||||
if local is None or local.integrity != "present":
|
||||
raise ValueError("selected item has no present local location")
|
||||
path = asset_dir / str(local.get("path") or "")
|
||||
expected = str(local.get("sha256") or "")
|
||||
filename = str(local.local_path) if local.local_path is not None else ""
|
||||
path = asset_dir / filename
|
||||
expected = local.sha256 or ""
|
||||
if not path.is_file() or not expected or hash_file(path) != expected:
|
||||
raise ValueError("selected local file failed hash verification")
|
||||
return path, expected
|
||||
@@ -94,30 +115,32 @@ def activate_fallback(
|
||||
upload: Callable[[Path], str],
|
||||
lookup: Callable[[str, str], str | None] | None = None,
|
||||
) -> FallbackResult:
|
||||
"""Activate a fallback URL for ``media_id``: upload (or reuse) and update manifest + note references."""
|
||||
if not confirm_origin_unavailable:
|
||||
raise ValueError("origin-unavailable confirmation is required")
|
||||
media_path = asset_dir / "media.json"
|
||||
manifest = json.loads(media_path.read_text(encoding="utf-8"))
|
||||
raw = json.loads(media_path.read_text(encoding="utf-8"))
|
||||
manifest: MediaManifest = _from_wire_dict(raw)
|
||||
group = _group(manifest, media_id)
|
||||
original = _one(group, "orig")
|
||||
target = _one(group, role)
|
||||
origin = find_location(original, "origin:x")
|
||||
if origin is None:
|
||||
raise ValueError("original item has no X origin location")
|
||||
old_url = selected_url(next((item for item in group if item.get("embed")), original))
|
||||
visible = next((item for item in group if item.embed), original)
|
||||
old_url = selected_url(visible)
|
||||
if old_url is None:
|
||||
raise ValueError("media group has no selected HTTPS location")
|
||||
local_path, content_hash = _verified_local(target, asset_dir)
|
||||
origin["availability"] = "unavailable"
|
||||
origin["confirmed_unavailable_at"] = now
|
||||
target = _with_origin_unavailable(target, origin, now)
|
||||
|
||||
fallback = next(
|
||||
(
|
||||
location
|
||||
for location in target.get("locations") or []
|
||||
if location.get("kind") == "fallback"
|
||||
and location.get("provider") == provider
|
||||
and location.get("sha256") == content_hash
|
||||
for location in target.locations
|
||||
if location.kind == "fallback"
|
||||
and location.provider == provider
|
||||
and location.sha256 == content_hash
|
||||
),
|
||||
None,
|
||||
)
|
||||
@@ -132,7 +155,7 @@ def activate_fallback(
|
||||
recorded_at=now,
|
||||
uploaded_at=None,
|
||||
)
|
||||
target.setdefault("locations", []).append(fallback)
|
||||
target = _append_location(target, fallback)
|
||||
reused = True
|
||||
if fallback is None:
|
||||
url = upload(local_path)
|
||||
@@ -143,19 +166,141 @@ def activate_fallback(
|
||||
recorded_at=now,
|
||||
uploaded_at=now,
|
||||
)
|
||||
target.setdefault("locations", []).append(fallback)
|
||||
target = _append_location(target, fallback)
|
||||
|
||||
atomic_write_json(media_path, manifest)
|
||||
replace_selected_url(note_dir, old_url, str(fallback["url"]))
|
||||
for item in group:
|
||||
item["embed"] = item is target
|
||||
target["publication"] = {
|
||||
"selected_location_id": fallback["id"],
|
||||
"selected_at": now,
|
||||
"reason": "origin-unavailable",
|
||||
manifest = _with_replaced_item(manifest, target)
|
||||
atomic_write_json(media_path, _manifest_to_wire_shape(manifest, raw))
|
||||
assert fallback.url is not None
|
||||
replace_selected_url(note_dir, old_url, fallback.url)
|
||||
manifest = _with_item_attr(manifest, target.post_id, target.media_id, "embed", True)
|
||||
manifest = _with_publication(
|
||||
manifest,
|
||||
target,
|
||||
Publication(
|
||||
selected_location_id=fallback.id or "",
|
||||
selected_at=now,
|
||||
reason="origin-unavailable",
|
||||
),
|
||||
)
|
||||
atomic_write_json(media_path, _manifest_to_wire_shape(manifest, raw))
|
||||
return FallbackResult(str(fallback.url or ""), reused)
|
||||
|
||||
|
||||
def _with_origin_unavailable(item: MediaItem, origin: MediaLocation, now: str) -> MediaItem:
|
||||
"""Return a copy of `item` with the matching origin location marked unavailable."""
|
||||
new_locations: list[MediaLocation] = []
|
||||
for location in item.locations:
|
||||
if location.id == origin.id:
|
||||
new_locations.append(
|
||||
MediaLocation(
|
||||
kind=location.kind,
|
||||
id=location.id,
|
||||
local_path=location.local_path,
|
||||
url=location.url,
|
||||
bytes=location.bytes,
|
||||
sha256=location.sha256,
|
||||
media_type=location.media_type,
|
||||
integrity=location.integrity,
|
||||
verified_at=location.verified_at,
|
||||
provider=location.provider,
|
||||
availability="unavailable",
|
||||
checked_at=location.checked_at,
|
||||
checked_status=location.checked_status,
|
||||
recorded_at=location.recorded_at,
|
||||
uploaded_at=location.uploaded_at,
|
||||
confirmed_unavailable_at=now,
|
||||
check=location.check,
|
||||
)
|
||||
)
|
||||
else:
|
||||
new_locations.append(location)
|
||||
return _copy_item_with(item, locations=tuple(new_locations))
|
||||
|
||||
|
||||
def _append_location(item: MediaItem, location: MediaLocation) -> MediaItem:
|
||||
return _copy_item_with(item, locations=item.locations + (location,))
|
||||
|
||||
|
||||
def _copy_item_with(item: MediaItem, **changes: Any) -> MediaItem:
|
||||
return MediaItem(
|
||||
post_id=changes.get("post_id", item.post_id),
|
||||
media_id=changes.get("media_id", item.media_id),
|
||||
kind=changes.get("kind", item.kind),
|
||||
role=changes.get("role", item.role),
|
||||
handle=changes.get("handle", item.handle),
|
||||
embed=changes.get("embed", item.embed),
|
||||
caption=changes.get("caption", item.caption),
|
||||
derived_from=changes.get("derived_from", item.derived_from),
|
||||
publication=changes.get("publication", item.publication),
|
||||
locations=changes.get("locations", item.locations),
|
||||
)
|
||||
|
||||
|
||||
def _with_replaced_item(manifest: MediaManifest, replacement: MediaItem) -> MediaManifest:
|
||||
new_items: list[MediaItem] = []
|
||||
for existing in manifest.items:
|
||||
if existing.post_id == replacement.post_id and existing.media_id == replacement.media_id:
|
||||
new_items.append(replacement)
|
||||
else:
|
||||
new_items.append(existing)
|
||||
return MediaManifest(
|
||||
root_post_id=manifest.root_post_id,
|
||||
items=tuple(new_items),
|
||||
schema_version=manifest.schema_version,
|
||||
captured_at=manifest.captured_at,
|
||||
)
|
||||
|
||||
|
||||
def _with_item_attr(
|
||||
manifest: MediaManifest,
|
||||
post_id: str,
|
||||
media_id: str,
|
||||
attr: str,
|
||||
value: Any,
|
||||
) -> MediaManifest:
|
||||
new_items: list[MediaItem] = []
|
||||
for existing in manifest.items:
|
||||
if existing.post_id == post_id and existing.media_id == media_id:
|
||||
new_items.append(_copy_item_with(existing, **{attr: value}))
|
||||
else:
|
||||
new_items.append(existing)
|
||||
return MediaManifest(
|
||||
root_post_id=manifest.root_post_id,
|
||||
items=tuple(new_items),
|
||||
schema_version=manifest.schema_version,
|
||||
captured_at=manifest.captured_at,
|
||||
)
|
||||
|
||||
|
||||
def _with_publication(
|
||||
manifest: MediaManifest,
|
||||
item: MediaItem,
|
||||
publication: Publication,
|
||||
) -> MediaManifest:
|
||||
new_items: list[MediaItem] = []
|
||||
for existing in manifest.items:
|
||||
if existing.post_id == item.post_id and existing.media_id == item.media_id:
|
||||
new_items.append(_copy_item_with(existing, publication=publication))
|
||||
else:
|
||||
new_items.append(existing)
|
||||
return MediaManifest(
|
||||
root_post_id=manifest.root_post_id,
|
||||
items=tuple(new_items),
|
||||
schema_version=manifest.schema_version,
|
||||
captured_at=manifest.captured_at,
|
||||
)
|
||||
|
||||
|
||||
def _manifest_to_wire_shape(manifest: MediaManifest, raw: dict) -> dict:
|
||||
out: dict = {
|
||||
"schema_version": manifest.schema_version or 2,
|
||||
"root_post_id": manifest.root_post_id,
|
||||
"items": [_item_to_wire(item) for item in manifest.items],
|
||||
"mirrors": list(raw.get("mirrors") or []),
|
||||
}
|
||||
atomic_write_json(media_path, manifest)
|
||||
return FallbackResult(str(fallback["url"]), reused)
|
||||
if manifest.captured_at is not None:
|
||||
out["captured_at"] = manifest.captured_at
|
||||
return out
|
||||
|
||||
|
||||
def find_existing_fallback(
|
||||
@@ -163,20 +308,23 @@ def find_existing_fallback(
|
||||
provider: str,
|
||||
content_hash: str,
|
||||
) -> str | None:
|
||||
"""Scan every thread's media.json for a fallback URL with matching sha256; returns the first or ``None``."""
|
||||
urls: set[str] = set()
|
||||
urls: set[str] = set()
|
||||
for media_path in assets_root.rglob("media.json"):
|
||||
data = json.loads(media_path.read_text(encoding="utf-8"))
|
||||
if data.get("schema_version") != 2:
|
||||
raw = json.loads(media_path.read_text(encoding="utf-8"))
|
||||
if raw.get("schema_version") != 2:
|
||||
continue
|
||||
for item in data.get("items") or []:
|
||||
for location in item.get("locations") or []:
|
||||
manifest = _from_wire_dict(raw)
|
||||
for item in manifest.items:
|
||||
for location in item.locations:
|
||||
if (
|
||||
location.get("kind") == "fallback"
|
||||
and location.get("provider") == provider
|
||||
and location.get("sha256") == content_hash
|
||||
and str(location.get("url") or "").startswith("https://")
|
||||
location.kind == "fallback"
|
||||
and location.provider == provider
|
||||
and location.sha256 == content_hash
|
||||
and (location.url or "").startswith("https://")
|
||||
):
|
||||
urls.add(str(location["url"]))
|
||||
urls.add(location.url or "")
|
||||
return sorted(urls)[0] if urls else None
|
||||
|
||||
|
||||
@@ -187,31 +335,74 @@ def restore_origin(
|
||||
media_id: str,
|
||||
now: str,
|
||||
) -> None:
|
||||
"""Switch the visible location for ``media_id`` back to its X origin and rewrite the note references."""
|
||||
media_path = asset_dir / "media.json"
|
||||
manifest = json.loads(media_path.read_text(encoding="utf-8"))
|
||||
media_path = asset_dir / "media.json"
|
||||
raw = json.loads(media_path.read_text(encoding="utf-8"))
|
||||
manifest = _from_wire_dict(raw)
|
||||
group = _group(manifest, media_id)
|
||||
original = _one(group, "orig")
|
||||
origin = find_location(original, "origin:x")
|
||||
if origin is None:
|
||||
raise ValueError("original item has no X origin location")
|
||||
visible = next((item for item in group if item.get("embed")), original)
|
||||
visible = next((item for item in group if item.embed), original)
|
||||
old_url = selected_url(visible)
|
||||
new_url = str(origin.get("url") or "")
|
||||
new_url = origin.url or ""
|
||||
if old_url is None or not new_url.startswith("https://"):
|
||||
raise ValueError("media group has no restorable origin")
|
||||
replace_selected_url(note_dir, old_url, new_url)
|
||||
origin["availability"] = "available"
|
||||
origin["checked_at"] = now
|
||||
origin["check"] = {
|
||||
"status": None,
|
||||
"result": "available",
|
||||
"detail": "manual restore-origin selection",
|
||||
}
|
||||
for item in group:
|
||||
item["embed"] = item is original
|
||||
original["publication"] = {
|
||||
"selected_location_id": "origin:x",
|
||||
"selected_at": now,
|
||||
"reason": "manual",
|
||||
}
|
||||
atomic_write_json(media_path, manifest)
|
||||
new_locations: list[MediaLocation] = []
|
||||
for location in original.locations:
|
||||
if location.id == origin.id:
|
||||
new_locations.append(
|
||||
MediaLocation(
|
||||
kind=location.kind,
|
||||
id=location.id,
|
||||
local_path=location.local_path,
|
||||
url=location.url,
|
||||
bytes=location.bytes,
|
||||
sha256=location.sha256,
|
||||
media_type=location.media_type,
|
||||
integrity=location.integrity,
|
||||
verified_at=location.verified_at,
|
||||
provider=location.provider,
|
||||
availability="available",
|
||||
checked_at=now,
|
||||
checked_status=location.checked_status,
|
||||
recorded_at=location.recorded_at,
|
||||
uploaded_at=location.uploaded_at,
|
||||
confirmed_unavailable_at=location.confirmed_unavailable_at,
|
||||
check=MediaLocationCheck(
|
||||
status=None,
|
||||
result="available",
|
||||
detail="manual restore-origin selection",
|
||||
),
|
||||
)
|
||||
)
|
||||
else:
|
||||
new_locations.append(location)
|
||||
manifest = _with_replaced_item(
|
||||
manifest, _copy_item_with(original, locations=tuple(new_locations))
|
||||
)
|
||||
new_items: list[MediaItem] = []
|
||||
for existing in manifest.items:
|
||||
if existing.post_id == original.post_id and existing.media_id == original.media_id:
|
||||
new_items.append(_copy_item_with(existing, embed=True))
|
||||
else:
|
||||
new_items.append(_copy_item_with(existing, embed=False))
|
||||
manifest = MediaManifest(
|
||||
root_post_id=manifest.root_post_id,
|
||||
items=tuple(new_items),
|
||||
schema_version=manifest.schema_version,
|
||||
captured_at=manifest.captured_at,
|
||||
)
|
||||
manifest = _with_publication(
|
||||
manifest,
|
||||
original,
|
||||
Publication(
|
||||
selected_location_id="origin:x",
|
||||
selected_at=now,
|
||||
reason="manual",
|
||||
),
|
||||
)
|
||||
atomic_write_json(media_path, _manifest_to_wire_shape(manifest, raw))
|
||||
Reference in new issue
Block a user