another test of refresh with diff op.
This commit is contained in:
1 parent
7e4ed4b303
commit
3805685d8f
10 files changed
+640
-73
No files matched your search
@@ -2,11 +2,13 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Self
|
||||
import json
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PostMetrics:
|
||||
"""Post engagement counters (replies / reposts / likes / views)."""
|
||||
reply_count: int
|
||||
repost_count: int
|
||||
like_count: int
|
||||
@@ -15,6 +17,7 @@ class PostMetrics:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PostData:
|
||||
"""A single tweet post extracted from a Twitter archive."""
|
||||
post_id: str
|
||||
author: str
|
||||
handle: str
|
||||
@@ -28,11 +31,149 @@ class PostData:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ThreadData:
|
||||
"""A complete Twitter thread (root post + replies)."""
|
||||
root_post_id: str
|
||||
posts: tuple[PostData, ...]
|
||||
source_url: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MediaLocation:
|
||||
"""A single media location record (local / remote / fallback / mirror / derived)."""
|
||||
kind: str
|
||||
local_path: Path | None = None
|
||||
url: str | None = None
|
||||
bytes: int | None = None
|
||||
sha256: str | None = None
|
||||
checked_at: str | None = None
|
||||
checked_status: int | None = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, raw: dict[str, Any]) -> Self:
|
||||
if "kind" not in raw:
|
||||
raise KeyError("MediaLocation.from_dict: 'kind' is required")
|
||||
return cls(
|
||||
kind=_required_str(raw["kind"], "kind"),
|
||||
local_path=_optional_path(raw.get("local_path"), "local_path"),
|
||||
url=_optional_str(raw.get("url"), "url"),
|
||||
bytes=_optional_int(raw.get("bytes"), "bytes"),
|
||||
sha256=_optional_str(raw.get("sha256"), "sha256"),
|
||||
checked_at=_optional_str(raw.get("checked_at"), "checked_at"),
|
||||
checked_status=_optional_int(raw.get("checked_status"), "checked_status"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MediaItem:
|
||||
"""A single media entry attached to a post (image / crt / crt_outline / denoise / ocr)."""
|
||||
post_id: str
|
||||
media_id: str
|
||||
kind: str
|
||||
locations: tuple[MediaLocation, ...]
|
||||
embed: bool | None = None
|
||||
caption: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, raw: dict[str, Any]) -> Self:
|
||||
for required in ("post_id", "media_id", "kind", "locations"):
|
||||
if required not in raw:
|
||||
raise KeyError(f"MediaItem.from_dict: '{required}' is required")
|
||||
locations_raw = raw["locations"]
|
||||
if not isinstance(locations_raw, list):
|
||||
raise TypeError(
|
||||
f"MediaItem.from_dict: 'locations' must be list, got {type(locations_raw).__name__}"
|
||||
)
|
||||
return cls(
|
||||
post_id=_required_str(raw["post_id"], "post_id"),
|
||||
media_id=_required_str(raw["media_id"], "media_id"),
|
||||
kind=_required_str(raw["kind"], "kind"),
|
||||
locations=tuple(MediaLocation.from_dict(loc) for loc in locations_raw),
|
||||
embed=_optional_bool(raw.get("embed"), "embed"),
|
||||
caption=_optional_str(raw.get("caption"), "caption"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MediaManifest:
|
||||
"""A whole media.json document (one thread)."""
|
||||
root_post_id: str
|
||||
items: tuple[MediaItem, ...]
|
||||
captured_at: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, raw: dict[str, Any]) -> Self:
|
||||
for required in ("root_post_id", "items"):
|
||||
if required not in raw:
|
||||
raise KeyError(f"MediaManifest.from_dict: '{required}' is required")
|
||||
items_raw = raw["items"]
|
||||
if not isinstance(items_raw, list):
|
||||
raise TypeError(
|
||||
f"MediaManifest.from_dict: 'items' must be list, got {type(items_raw).__name__}"
|
||||
)
|
||||
return cls(
|
||||
root_post_id=_required_str(raw["root_post_id"], "root_post_id"),
|
||||
items=tuple(MediaItem.from_dict(item) for item in items_raw),
|
||||
captured_at=_optional_str(raw.get("captured_at"), "captured_at"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LegacyThreadData:
|
||||
"""Pre-v2 thread_data.json shape paired with media items during migration."""
|
||||
root_post_id: str
|
||||
posts: tuple[PostData, ...]
|
||||
source_url: str
|
||||
media_items: tuple[MediaItem, ...] = ()
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, raw: dict[str, Any]) -> Self:
|
||||
for required in ("root_post_id", "posts", "source_url"):
|
||||
if required not in raw:
|
||||
raise KeyError(f"LegacyThreadData.from_dict: '{required}' is required")
|
||||
posts_raw = raw["posts"]
|
||||
if not isinstance(posts_raw, list):
|
||||
raise TypeError(
|
||||
f"LegacyThreadData.from_dict: 'posts' must be list, got {type(posts_raw).__name__}"
|
||||
)
|
||||
media_items_raw = raw.get("media_items")
|
||||
media_items: tuple[MediaItem, ...] = ()
|
||||
if media_items_raw is not None:
|
||||
if not isinstance(media_items_raw, list):
|
||||
raise TypeError(
|
||||
f"LegacyThreadData.from_dict: 'media_items' must be list, got {type(media_items_raw).__name__}"
|
||||
)
|
||||
media_items = tuple(MediaItem.from_dict(item) for item in media_items_raw)
|
||||
return cls(
|
||||
root_post_id=_required_str(raw["root_post_id"], "root_post_id"),
|
||||
posts=tuple(_parse_post(p) for p in posts_raw),
|
||||
source_url=_required_str(raw["source_url"], "source_url"),
|
||||
media_items=media_items,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LegacyManifest:
|
||||
"""Pre-v2 corpus inventory shape used during media migration."""
|
||||
threads: tuple[LegacyThreadData, ...]
|
||||
schema_version: int | None = None
|
||||
generated_at: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, raw: dict[str, Any]) -> Self:
|
||||
if "threads" not in raw:
|
||||
raise KeyError("LegacyManifest.from_dict: 'threads' is required")
|
||||
threads_raw = raw["threads"]
|
||||
if not isinstance(threads_raw, list):
|
||||
raise TypeError(
|
||||
f"LegacyManifest.from_dict: 'threads' must be list, got {type(threads_raw).__name__}"
|
||||
)
|
||||
return cls(
|
||||
threads=tuple(LegacyThreadData.from_dict(t) for t in threads_raw),
|
||||
schema_version=_optional_int(raw.get("schema_version"), "schema_version"),
|
||||
generated_at=_optional_str(raw.get("generated_at"), "generated_at"),
|
||||
)
|
||||
|
||||
|
||||
def _norm_handle(raw: str) -> str:
|
||||
h = (raw or "").strip()
|
||||
return h[1:] if h.startswith("@") else h
|
||||
@@ -45,6 +186,76 @@ def _norm_id(raw: object) -> str | None:
|
||||
return s or None
|
||||
|
||||
|
||||
def _required_str(value: object, name: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(f"{name}: expected str, got {type(value).__name__}")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_str(value: object, field: str) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(f"{field}: expected str, got {type(value).__name__}")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_int(value: object, field: str) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise TypeError(f"{field}: expected int, got {type(value).__name__}")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_bool(value: object, field: str) -> bool | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, bool):
|
||||
raise TypeError(f"{field}: expected bool, got {type(value).__name__}")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_path(value: object, field: str) -> Path | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, Path):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return Path(value)
|
||||
raise TypeError(f"{field}: expected str or Path, got {type(value).__name__}")
|
||||
|
||||
|
||||
def _parse_post(raw: object) -> PostData:
|
||||
if not isinstance(raw, dict):
|
||||
raise TypeError(f"post entry: expected dict, got {type(raw).__name__}")
|
||||
m = raw.get("metrics") or {}
|
||||
if not isinstance(m, dict):
|
||||
raise TypeError(f"post entry metrics: expected dict, got {type(m).__name__}")
|
||||
view = m.get("view_count")
|
||||
media_urls_raw = raw.get("media_urls") or []
|
||||
if not isinstance(media_urls_raw, list):
|
||||
raise TypeError(
|
||||
f"post entry media_urls: expected list, got {type(media_urls_raw).__name__}"
|
||||
)
|
||||
return PostData(
|
||||
post_id=str(raw.get("post_id") or ""),
|
||||
author=str(raw.get("author") or ""),
|
||||
handle=_norm_handle(str(raw.get("handle") or "")),
|
||||
text=str(raw.get("text") or ""),
|
||||
timestamp=str(raw.get("timestamp") or ""),
|
||||
media_urls=tuple(str(u) for u in media_urls_raw),
|
||||
reply_to_id=_norm_id(raw.get("reply_to_id")),
|
||||
quote_of_id=_norm_id(raw.get("quote_of_id")),
|
||||
metrics=PostMetrics(
|
||||
reply_count=int(m.get("reply_count") or 0),
|
||||
repost_count=int(m.get("repost_count") or 0),
|
||||
like_count=int(m.get("like_count") or 0),
|
||||
view_count=int(view) if view is not None else None,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def load_thread(path: Path) -> ThreadData:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
posts: list[PostData] = []
|
||||
|
||||
Reference in new issue
Block a user