Make the scripts lesss bad python (still python...)
This commit is contained in:
1 parent
3805685d8f
commit
b64ab023ea
30 files changed
+1979
-653
No files matched your search
+262
-14
@@ -1,3 +1,4 @@
|
||||
"""Typed data model for the Twitter archive pipeline."""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
@@ -37,67 +38,192 @@ class ThreadData:
|
||||
source_url: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MediaLocationCheck:
|
||||
"""Origin/fallback URL HEAD probe outcome stored on a location."""
|
||||
status: int | None
|
||||
result: str
|
||||
detail: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MediaLocation:
|
||||
"""A single media location record (local / remote / fallback / mirror / derived)."""
|
||||
"""A single media location record (local / origin / fallback / mirror / derived)."""
|
||||
kind: str
|
||||
id: str | None = None
|
||||
local_path: Path | None = None
|
||||
url: str | None = None
|
||||
bytes: int | None = None
|
||||
sha256: str | None = None
|
||||
media_type: str | None = None
|
||||
integrity: str | None = None
|
||||
verified_at: str | None = None
|
||||
provider: str | None = None
|
||||
availability: str | None = None
|
||||
checked_at: str | None = None
|
||||
checked_status: int | None = None
|
||||
recorded_at: str | None = None
|
||||
uploaded_at: str | None = None
|
||||
confirmed_unavailable_at: str | None = None
|
||||
check: MediaLocationCheck | None = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, raw: dict[str, Any]) -> Self:
|
||||
if "kind" not in raw:
|
||||
raise KeyError("MediaLocation.from_dict: 'kind' is required")
|
||||
# Wire uses 'path' (basename) or 'local_path' (Path string) interchangeably
|
||||
path_raw = raw.get("local_path")
|
||||
if path_raw is None:
|
||||
path_raw = raw.get("path")
|
||||
check_raw = raw.get("check")
|
||||
check: MediaLocationCheck | None = None
|
||||
if check_raw is not None:
|
||||
if not isinstance(check_raw, dict):
|
||||
raise TypeError(
|
||||
f"check: expected dict, got {type(check_raw).__name__}"
|
||||
)
|
||||
if "result" not in check_raw or "detail" not in check_raw:
|
||||
raise KeyError("MediaLocation.from_dict: 'check' requires 'result' and 'detail'")
|
||||
check = MediaLocationCheck(
|
||||
status=_optional_int(check_raw.get("status"), "check.status"),
|
||||
result=_required_str(check_raw.get("result"), "check.result"),
|
||||
detail=_required_str(check_raw.get("detail"), "check.detail"),
|
||||
)
|
||||
return cls(
|
||||
kind=_required_str(raw["kind"], "kind"),
|
||||
local_path=_optional_path(raw.get("local_path"), "local_path"),
|
||||
id=_optional_str(raw.get("id"), "id"),
|
||||
local_path=_optional_path(path_raw, "local_path"),
|
||||
url=_optional_str(raw.get("url"), "url"),
|
||||
bytes=_optional_int(raw.get("bytes"), "bytes"),
|
||||
sha256=_optional_str(raw.get("sha256"), "sha256"),
|
||||
media_type=_optional_str(raw.get("media_type"), "media_type"),
|
||||
integrity=_optional_str(raw.get("integrity"), "integrity"),
|
||||
verified_at=_optional_str(raw.get("verified_at"), "verified_at"),
|
||||
provider=_optional_str(raw.get("provider"), "provider"),
|
||||
availability=_optional_str(raw.get("availability"), "availability"),
|
||||
checked_at=_optional_str(raw.get("checked_at"), "checked_at"),
|
||||
checked_status=_optional_int(raw.get("checked_status"), "checked_status"),
|
||||
recorded_at=_optional_str(raw.get("recorded_at"), "recorded_at"),
|
||||
uploaded_at=_optional_str(raw.get("uploaded_at"), "uploaded_at"),
|
||||
confirmed_unavailable_at=_optional_str(
|
||||
raw.get("confirmed_unavailable_at"), "confirmed_unavailable_at"
|
||||
),
|
||||
check=check,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MediaItem:
|
||||
"""A single media entry attached to a post (image / crt / crt_outline / denoise / ocr)."""
|
||||
class DerivedFromRef:
|
||||
"""Pointer to the parent (orig) media item for derived items (crt, ocr, etc.)."""
|
||||
post_id: str
|
||||
media_id: str
|
||||
kind: str
|
||||
locations: tuple[MediaLocation, ...]
|
||||
role: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Publication:
|
||||
"""Selected location for a media item; embedded as 'publication' on the wire."""
|
||||
selected_location_id: str
|
||||
selected_at: str
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MediaItem:
|
||||
"""A single media entry attached to a post (orig / crt / crt_outline / denoise / ocr).
|
||||
|
||||
For legacy (pre-v2) wire shapes, ``locations`` may be empty and ``filename``
|
||||
plus ``url`` carry the original on-disk filename and pre-migration fallback
|
||||
URL. v2 items leave both ``None`` and put the URL inside a ``MediaLocation``.
|
||||
"""
|
||||
post_id: str
|
||||
media_id: str
|
||||
locations: tuple[MediaLocation, ...] = ()
|
||||
kind: str | None = None
|
||||
handle: str | None = None
|
||||
role: str | None = None
|
||||
filename: str | None = None
|
||||
url: str | None = None
|
||||
embed: bool | None = None
|
||||
caption: str | None = None
|
||||
derived_from: DerivedFromRef | None = None
|
||||
publication: Publication | None = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, raw: dict[str, Any]) -> Self:
|
||||
for required in ("post_id", "media_id", "kind", "locations"):
|
||||
for required in ("post_id", "media_id"):
|
||||
if required not in raw:
|
||||
raise KeyError(f"MediaItem.from_dict: '{required}' is required")
|
||||
locations_raw = raw["locations"]
|
||||
if not isinstance(locations_raw, list):
|
||||
raise TypeError(
|
||||
f"MediaItem.from_dict: 'locations' must be list, got {type(locations_raw).__name__}"
|
||||
locations_raw = raw.get("locations")
|
||||
locations: tuple[MediaLocation, ...] = ()
|
||||
if locations_raw is not None:
|
||||
if not isinstance(locations_raw, list):
|
||||
raise TypeError(
|
||||
f"MediaItem.from_dict: 'locations' must be list, got {type(locations_raw).__name__}"
|
||||
)
|
||||
locations = tuple(MediaLocation.from_dict(loc) for loc in locations_raw)
|
||||
# Wire uses 'role' as the kind identifier; accept 'kind' for direct callers
|
||||
kind_raw = raw.get("role")
|
||||
if kind_raw is None:
|
||||
kind_raw = raw.get("kind")
|
||||
if kind_raw is None:
|
||||
raise KeyError("MediaItem.from_dict: 'role' or 'kind' is required")
|
||||
kind = _required_str(kind_raw, "role")
|
||||
derived_from_raw = raw.get("derived_from")
|
||||
derived_from: DerivedFromRef | None = None
|
||||
if derived_from_raw is not None:
|
||||
if not isinstance(derived_from_raw, dict):
|
||||
raise TypeError(
|
||||
f"derived_from: expected dict, got {type(derived_from_raw).__name__}"
|
||||
)
|
||||
derived_from = DerivedFromRef(
|
||||
post_id=_required_str(
|
||||
derived_from_raw.get("post_id"), "derived_from.post_id"
|
||||
),
|
||||
media_id=_required_str(
|
||||
derived_from_raw.get("media_id"), "derived_from.media_id"
|
||||
),
|
||||
role=_required_str(derived_from_raw.get("role"), "derived_from.role"),
|
||||
)
|
||||
publication_raw = raw.get("publication")
|
||||
publication: Publication | None = None
|
||||
if publication_raw is not None:
|
||||
if not isinstance(publication_raw, dict):
|
||||
raise TypeError(
|
||||
f"publication: expected dict, got {type(publication_raw).__name__}"
|
||||
)
|
||||
publication = Publication(
|
||||
selected_location_id=_required_str(
|
||||
publication_raw.get("selected_location_id"),
|
||||
"publication.selected_location_id",
|
||||
),
|
||||
selected_at=_required_str(
|
||||
publication_raw.get("selected_at"), "publication.selected_at"
|
||||
),
|
||||
reason=_required_str(publication_raw.get("reason"), "publication.reason"),
|
||||
)
|
||||
return cls(
|
||||
post_id=_required_str(raw["post_id"], "post_id"),
|
||||
media_id=_required_str(raw["media_id"], "media_id"),
|
||||
kind=_required_str(raw["kind"], "kind"),
|
||||
locations=tuple(MediaLocation.from_dict(loc) for loc in locations_raw),
|
||||
kind=kind,
|
||||
handle=_optional_str(raw.get("handle"), "handle"),
|
||||
role=_optional_str(raw.get("role"), "role"),
|
||||
filename=_optional_str(raw.get("filename"), "filename"),
|
||||
url=_optional_str(raw.get("url"), "url"),
|
||||
locations=locations,
|
||||
embed=_optional_bool(raw.get("embed"), "embed"),
|
||||
caption=_optional_str(raw.get("caption"), "caption"),
|
||||
derived_from=derived_from,
|
||||
publication=publication,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MediaManifest:
|
||||
"""A whole media.json document (one thread)."""
|
||||
"""A whole media.json document (one thread). schema_version is the wire key."""
|
||||
root_post_id: str
|
||||
items: tuple[MediaItem, ...]
|
||||
schema_version: int | None = None
|
||||
captured_at: str | None = None
|
||||
|
||||
@classmethod
|
||||
@@ -113,6 +239,7 @@ class MediaManifest:
|
||||
return cls(
|
||||
root_post_id=_required_str(raw["root_post_id"], "root_post_id"),
|
||||
items=tuple(MediaItem.from_dict(item) for item in items_raw),
|
||||
schema_version=_optional_int(raw.get("schema_version"), "schema_version"),
|
||||
captured_at=_optional_str(raw.get("captured_at"), "captured_at"),
|
||||
)
|
||||
|
||||
@@ -157,6 +284,8 @@ class LegacyManifest:
|
||||
threads: tuple[LegacyThreadData, ...]
|
||||
schema_version: int | None = None
|
||||
generated_at: str | None = None
|
||||
counts: dict[str, int] | None = None
|
||||
frozen_hashes: dict[str, str] | None = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, raw: dict[str, Any]) -> Self:
|
||||
@@ -167,13 +296,132 @@ class LegacyManifest:
|
||||
raise TypeError(
|
||||
f"LegacyManifest.from_dict: 'threads' must be list, got {type(threads_raw).__name__}"
|
||||
)
|
||||
counts: dict[str, int] | None = _coerce_str_int_map(raw.get("counts"), "counts")
|
||||
frozen_hashes: dict[str, str] | None = _coerce_str_str_map(
|
||||
raw.get("frozen_hashes"), "frozen_hashes"
|
||||
)
|
||||
return cls(
|
||||
threads=tuple(LegacyThreadData.from_dict(t) for t in threads_raw),
|
||||
schema_version=_optional_int(raw.get("schema_version"), "schema_version"),
|
||||
generated_at=_optional_str(raw.get("generated_at"), "generated_at"),
|
||||
counts=counts,
|
||||
frozen_hashes=frozen_hashes,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OriginCheck:
|
||||
"""Outcome of a HEAD probe against a media item's origin URL.
|
||||
|
||||
``status`` is the HTTP status code returned by the probe (or ``None`` if
|
||||
the request errored before a response arrived). ``result`` is the
|
||||
classifier label (``"available"`` for 2xx/3xx, ``"error"`` otherwise).
|
||||
``confirms_unavailable`` distinguishes transient failures like 429 from
|
||||
definitive unavailability so the caller does not flip the location's
|
||||
availability state on a rate-limit blip.
|
||||
"""
|
||||
checked_at: str
|
||||
status: int | None
|
||||
result: str
|
||||
detail: str
|
||||
confirms_unavailable: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LegacyMediaJson:
|
||||
"""Per-thread ``media.json`` shape consumed by ``media_migrate.py`` and ``media_audit.py``.
|
||||
|
||||
Holds both the pre-v2 (legacy) and v2 (canonical) wire shapes so the
|
||||
migration and audit entry points can share one boundary type. The
|
||||
``items`` field is parsed as ``MediaItem`` instances; for legacy items
|
||||
``locations`` is empty and ``filename`` plus ``url`` carry the
|
||||
on-disk filename and pre-migration fallback URL.
|
||||
|
||||
``mirrors`` hold opaque destination/auth fields that are intentionally
|
||||
not yet typed. This is the only non-typed dict in the data model — it
|
||||
is consumed only by iteration, never by attribute access, and we
|
||||
deliberately do not commit to a structure until the backup subsystem
|
||||
settles on a wire shape.
|
||||
"""
|
||||
schema_version: int | None
|
||||
root_post_id: str
|
||||
items: tuple[MediaItem, ...]
|
||||
captured_at: str | None
|
||||
mirrors: dict[str, Any] | None = None
|
||||
frozen: bool | None = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, raw: dict[str, Any]) -> Self:
|
||||
if "root_post_id" not in raw:
|
||||
raise KeyError("LegacyMediaJson.from_dict: 'root_post_id' is required")
|
||||
items_raw = raw.get("items")
|
||||
if items_raw is None:
|
||||
items_raw = []
|
||||
if not isinstance(items_raw, list):
|
||||
raise TypeError(
|
||||
f"LegacyMediaJson.from_dict: 'items' must be list, got {type(items_raw).__name__}"
|
||||
)
|
||||
mirrors = _coerce_mirrors(raw.get("mirrors"), "mirrors")
|
||||
return cls(
|
||||
schema_version=_optional_int(raw.get("schema_version"), "schema_version"),
|
||||
root_post_id=_required_str(raw["root_post_id"], "root_post_id"),
|
||||
items=tuple(MediaItem.from_dict(item) for item in items_raw),
|
||||
captured_at=_optional_str(raw.get("captured_at"), "captured_at"),
|
||||
mirrors=mirrors,
|
||||
frozen=_optional_bool(raw.get("frozen"), "frozen"),
|
||||
)
|
||||
|
||||
|
||||
def _coerce_str_int_map(value: object, field: str) -> dict[str, int] | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, dict):
|
||||
raise TypeError(f"{field}: expected dict, got {type(value).__name__}")
|
||||
out: dict[str, int] = {}
|
||||
for key, item in value.items():
|
||||
if isinstance(item, bool) or not isinstance(item, int):
|
||||
raise TypeError(f"{field}[{key}]: expected int")
|
||||
out[str(key)] = item
|
||||
return out
|
||||
|
||||
|
||||
def _coerce_str_str_map(value: object, field: str) -> dict[str, str] | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, dict):
|
||||
raise TypeError(f"{field}: expected dict, got {type(value).__name__}")
|
||||
out: dict[str, str] = {}
|
||||
for key, item in value.items():
|
||||
if not isinstance(item, str):
|
||||
raise TypeError(f"{field}[{key}]: expected str")
|
||||
out[str(key)] = item
|
||||
return out
|
||||
|
||||
|
||||
def _coerce_mirrors(value: object, field: str) -> dict[str, Any] | None:
|
||||
"""Accept the legacy ``mirrors: list[dict]`` wire and coerce to the typed mirrors shape.
|
||||
|
||||
Legacy producers (pre-LegacyMediaJson) wrote ``mirrors`` as a list of
|
||||
destination entries; current producers may write it as a dict keyed by
|
||||
destination id. Both shapes are accepted; the returned dict is keyed
|
||||
by ``destination_id`` when present on each entry, falling back to the
|
||||
string index for entries without one.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
if isinstance(value, list):
|
||||
coerced: dict = {}
|
||||
for index, entry in enumerate(value):
|
||||
if isinstance(entry, dict) and "destination_id" in entry:
|
||||
coerced[str(entry["destination_id"])] = entry
|
||||
else:
|
||||
coerced[str(index)] = entry
|
||||
return coerced
|
||||
raise TypeError(f"{field}: expected dict or list, got {type(value).__name__}")
|
||||
|
||||
|
||||
def _norm_handle(raw: str) -> str:
|
||||
h = (raw or "").strip()
|
||||
return h[1:] if h.startswith("@") else h
|
||||
|
||||
Reference in new issue
Block a user