337 lines
11 KiB
Python
337 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
from datetime import datetime, timezone
|
|
from hashlib import sha256
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
SCHEMA_VERSION = 2
|
|
VISUAL_ROLES = {"orig", "crt", "crt_outline", "denoise"}
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def hash_file(path: Path) -> str:
|
|
digest = sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def atomic_write_json(path: Path, data: dict) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
|
try:
|
|
with temp.open("w", encoding="utf-8", newline="\n") as handle:
|
|
json.dump(data, handle, indent=2, ensure_ascii=False)
|
|
handle.write("\n")
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temp, path)
|
|
finally:
|
|
if temp.exists():
|
|
temp.unlink()
|
|
|
|
|
|
def item_key(item: dict[str, Any]) -> tuple[str, str, str]:
|
|
return (
|
|
str(item.get("post_id") or ""),
|
|
str(item.get("media_id") or ""),
|
|
str(item.get("role") or ""),
|
|
)
|
|
|
|
|
|
def find_location(item: dict[str, Any], location_id: str) -> dict[str, Any] | None:
|
|
for location in item.get("locations") or []:
|
|
if str(location.get("id") or "") == location_id:
|
|
return location
|
|
return None
|
|
|
|
|
|
def location_of_kind(item: dict[str, Any], kind: str) -> dict[str, Any] | None:
|
|
for location in item.get("locations") or []:
|
|
if location.get("kind") == kind:
|
|
return location
|
|
return None
|
|
|
|
|
|
def selected_location(item: dict[str, Any]) -> dict[str, Any] | None:
|
|
publication = item.get("publication")
|
|
if not isinstance(publication, dict):
|
|
return None
|
|
return find_location(item, str(publication.get("selected_location_id") or ""))
|
|
|
|
|
|
def selected_url(item: dict[str, Any]) -> str | None:
|
|
location = selected_location(item)
|
|
if not location or location.get("kind") not in {"origin", "fallback"}:
|
|
return None
|
|
value = str(location.get("url") or "")
|
|
return value if value.startswith("https://") else None
|
|
|
|
|
|
def _media_type(filename: str) -> str:
|
|
guessed, _encoding = mimetypes.guess_type(filename)
|
|
return guessed or "application/octet-stream"
|
|
|
|
|
|
def _local_location(filename: str, local_path: Path, now: str) -> dict[str, Any]:
|
|
present = local_path.is_file()
|
|
return {
|
|
"id": "local",
|
|
"kind": "local",
|
|
"path": filename,
|
|
"sha256": hash_file(local_path) if present else None,
|
|
"bytes": local_path.stat().st_size if present else None,
|
|
"media_type": _media_type(filename),
|
|
"integrity": "present" if present else "missing",
|
|
"verified_at": now,
|
|
}
|
|
|
|
|
|
def new_original_item(
|
|
*,
|
|
post_id: str,
|
|
media_id: str,
|
|
handle: str,
|
|
origin_url: str,
|
|
filename: str,
|
|
local_path: Path,
|
|
now: str,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"post_id": post_id,
|
|
"media_id": media_id,
|
|
"handle": handle,
|
|
"role": "orig",
|
|
"derived_from": None,
|
|
"embed": True,
|
|
"locations": [
|
|
{
|
|
"id": "origin:x",
|
|
"kind": "origin",
|
|
"provider": "x",
|
|
"url": origin_url,
|
|
"availability": "unknown",
|
|
"checked_at": None,
|
|
"check": None,
|
|
"confirmed_unavailable_at": None,
|
|
},
|
|
_local_location(filename, local_path, now),
|
|
],
|
|
"publication": {
|
|
"selected_location_id": "origin:x",
|
|
"selected_at": now,
|
|
"reason": "default",
|
|
},
|
|
}
|
|
|
|
|
|
def merge_item(existing: dict[str, Any], fresh: dict[str, Any]) -> dict[str, Any]:
|
|
if item_key(existing) != item_key(fresh):
|
|
raise ValueError("cannot merge different media items")
|
|
merged = copy.deepcopy(existing)
|
|
merged["handle"] = fresh["handle"]
|
|
existing_origin = location_of_kind(existing, "origin")
|
|
fresh_origin = location_of_kind(fresh, "origin")
|
|
if (
|
|
existing_origin is not None
|
|
and fresh_origin is not None
|
|
and existing_origin.get("url") != fresh_origin.get("url")
|
|
):
|
|
raise ValueError(f"origin URL changed for {item_key(existing)!r}")
|
|
fresh_local = location_of_kind(fresh, "local")
|
|
locations = [
|
|
copy.deepcopy(location)
|
|
for location in merged.get("locations") or []
|
|
if location.get("kind") != "local"
|
|
]
|
|
if fresh_local is not None:
|
|
locations.append(copy.deepcopy(fresh_local))
|
|
if location_of_kind(merged, "origin") is None:
|
|
fresh_origin = location_of_kind(fresh, "origin")
|
|
if fresh_origin is not None:
|
|
locations.insert(0, copy.deepcopy(fresh_origin))
|
|
merged["locations"] = locations
|
|
if merged.get("publication") is None:
|
|
merged["publication"] = copy.deepcopy(fresh.get("publication"))
|
|
return merged
|
|
|
|
|
|
def merge_manifest_items(
|
|
existing_items: list[dict[str, Any]],
|
|
fresh_items: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
existing_by_key = {item_key(item): item for item in existing_items}
|
|
fresh_keys: set[tuple[str, str, str]] = set()
|
|
merged: list[dict[str, Any]] = []
|
|
for fresh in fresh_items:
|
|
key = item_key(fresh)
|
|
fresh_keys.add(key)
|
|
previous = existing_by_key.get(key)
|
|
merged.append(merge_item(previous, fresh) if previous is not None else fresh)
|
|
for previous in existing_items:
|
|
if item_key(previous) not in fresh_keys:
|
|
merged.append(copy.deepcopy(previous))
|
|
return merged
|
|
|
|
|
|
def fallback_location_id(provider: str, url: str) -> str:
|
|
digest = sha256(f"{provider}\0{url}".encode("utf-8")).hexdigest()[:16]
|
|
return f"fallback:{provider}:{digest}"
|
|
|
|
|
|
def new_fallback_location(
|
|
*,
|
|
provider: str,
|
|
url: str,
|
|
content_hash: str | None,
|
|
recorded_at: str,
|
|
uploaded_at: str | None,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"id": fallback_location_id(provider, url),
|
|
"kind": "fallback",
|
|
"provider": provider,
|
|
"url": url,
|
|
"sha256": content_hash,
|
|
"recorded_at": recorded_at,
|
|
"uploaded_at": uploaded_at,
|
|
"availability": "unknown",
|
|
"checked_at": None,
|
|
"check": None,
|
|
}
|
|
|
|
|
|
def new_derived_item(
|
|
*,
|
|
legacy: dict[str, Any],
|
|
asset_dir: Path,
|
|
now: str,
|
|
) -> dict[str, Any]:
|
|
filename = str(legacy.get("filename") or "")
|
|
post_id = str(legacy.get("post_id") or "")
|
|
media_id = str(legacy.get("media_id") or "")
|
|
role = str(legacy.get("role") or "")
|
|
return {
|
|
"post_id": post_id,
|
|
"media_id": media_id,
|
|
"handle": str(legacy.get("handle") or ""),
|
|
"role": role,
|
|
"derived_from": {
|
|
"post_id": post_id,
|
|
"media_id": media_id,
|
|
"role": "orig",
|
|
},
|
|
"embed": False,
|
|
"locations": [_local_location(filename, asset_dir / filename, now)],
|
|
"publication": None,
|
|
}
|
|
|
|
|
|
def upsert_derived_item(
|
|
manifest: dict[str, Any],
|
|
*,
|
|
post_id: str,
|
|
media_id: str,
|
|
handle: str,
|
|
role: str,
|
|
filename: str,
|
|
asset_dir: Path,
|
|
now: str,
|
|
) -> dict[str, Any]:
|
|
legacy_shape = {
|
|
"post_id": post_id,
|
|
"media_id": media_id,
|
|
"handle": handle,
|
|
"role": role,
|
|
"filename": filename,
|
|
}
|
|
fresh = new_derived_item(legacy=legacy_shape, asset_dir=asset_dir, now=now)
|
|
key = item_key(fresh)
|
|
items = list(manifest.get("items") or [])
|
|
for index, existing in enumerate(items):
|
|
if item_key(existing) == key:
|
|
fresh["locations"] = [
|
|
location
|
|
for location in existing.get("locations") or []
|
|
if location.get("kind") != "local"
|
|
] + fresh["locations"]
|
|
fresh["embed"] = bool(existing.get("embed"))
|
|
fresh["publication"] = copy.deepcopy(existing.get("publication"))
|
|
items[index] = fresh
|
|
manifest["items"] = items
|
|
return fresh
|
|
items.append(fresh)
|
|
manifest["items"] = items
|
|
return fresh
|
|
|
|
|
|
def canonical_manifest_bytes(data: dict[str, Any]) -> bytes:
|
|
stable = copy.deepcopy(data)
|
|
stable["mirrors"] = []
|
|
return (
|
|
json.dumps(stable, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
|
+ "\n"
|
|
).encode("utf-8")
|
|
|
|
|
|
def payload_inventory(asset_dir: Path, manifest: dict[str, Any]) -> dict[str, str]:
|
|
inventory: dict[str, str] = {
|
|
"media.json#canonical": sha256(canonical_manifest_bytes(manifest)).hexdigest()
|
|
}
|
|
for path in sorted(asset_dir.iterdir(), key=lambda candidate: candidate.name):
|
|
if path.is_file() and path.name != "media.json":
|
|
inventory[path.name] = hash_file(path)
|
|
return inventory
|
|
|
|
|
|
def inventory_digest(inventory: dict[str, str]) -> str:
|
|
encoded = json.dumps(inventory, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
return sha256(encoded).hexdigest()
|
|
|
|
|
|
def validate_manifest(data: dict[str, Any]) -> list[str]:
|
|
issues: list[str] = []
|
|
if data.get("schema_version") != SCHEMA_VERSION:
|
|
issues.append(f"schema_version must be {SCHEMA_VERSION}")
|
|
if not str(data.get("root_post_id") or ""):
|
|
issues.append("root_post_id is empty")
|
|
seen: set[tuple[str, str, str]] = set()
|
|
for index, item in enumerate(data.get("items") or []):
|
|
key = item_key(item)
|
|
if not all(key):
|
|
issues.append(f"items[{index}] has incomplete key {key!r}")
|
|
if key in seen:
|
|
issues.append(f"items[{index}] duplicates key {key!r}")
|
|
seen.add(key)
|
|
location_ids: set[str] = set()
|
|
for location in item.get("locations") or []:
|
|
location_id = str(location.get("id") or "")
|
|
if not location_id:
|
|
issues.append(f"items[{index}] has empty location id")
|
|
elif location_id in location_ids:
|
|
issues.append(f"items[{index}] duplicates location {location_id}")
|
|
location_ids.add(location_id)
|
|
publication = item.get("publication")
|
|
if publication is None:
|
|
if item.get("role") in VISUAL_ROLES and item.get("embed"):
|
|
issues.append(f"items[{index}] embeds without publication")
|
|
continue
|
|
selected_id = str(publication.get("selected_location_id") or "")
|
|
location = find_location(item, selected_id)
|
|
if location is None:
|
|
issues.append(f"items[{index}] selects missing location {selected_id}")
|
|
elif location.get("kind") not in {"origin", "fallback"}:
|
|
issues.append(f"items[{index}] selects non-HTTPS location {selected_id}")
|
|
elif not str(location.get("url") or "").startswith("https://"):
|
|
issues.append(f"items[{index}] selects non-HTTPS location {selected_id}")
|
|
return issues |