Make the scripts lesss bad python (still python...)

This commit is contained in:
ed committed 2026-08-24 16:47:03 -04:00
1 parent 3805685d8f
commit b64ab023ea
30 files changed
+1979 -653

No files matched your search

+2 -1
View File
@@ -1 +1,2 @@
"""Isolated tests for Threadwell Twitter tooling."""
"""Isolated tests for Threadwell Twitter tooling."""
from __future__ import annotations
+5 -3
View File
@@ -7,6 +7,7 @@ from pathlib import Path
from twitter.backup_assets import backup_thread
from twitter.media_audit import audit_thread
from twitter.media_manifest import (
_from_wire_dict,
canonical_manifest_bytes,
hash_file,
inventory_digest,
@@ -74,14 +75,15 @@ class BackupTests(unittest.TestCase):
now=NOW,
)
manifest = read_json(asset_dir / "media.json")
inventory = payload_inventory(asset_dir, manifest)
typed_manifest = _from_wire_dict(manifest)
inventory = payload_inventory(asset_dir, typed_manifest)
self.assertEqual(
manifest["mirrors"][0]["inventory_digest"],
inventory_digest(inventory),
)
backup_inventory_before = inventory_digest(payload_inventory(asset_dir, manifest))
backup_inventory_before = inventory_digest(payload_inventory(asset_dir, typed_manifest))
(asset_dir / "100_AAA_orig.png").write_bytes(b"modified")
new_inventory = payload_inventory(asset_dir, manifest)
new_inventory = payload_inventory(asset_dir, typed_manifest)
self.assertNotEqual(
inventory_digest(new_inventory),
backup_inventory_before,
+14 -9
View File
@@ -5,7 +5,7 @@ import unittest
from pathlib import Path
from twitter.fallback_media import activate_fallback, restore_origin
from twitter.media_manifest import selected_url
from twitter.media_manifest import _from_wire_dict, selected_url
from twitter.media_migrate import migrate_legacy_thread
from twitter.tests.helpers import NOW, legacy_thread, read_json
@@ -34,8 +34,8 @@ class FallbackTests(unittest.TestCase):
)
self.assertEqual(calls, [])
self.assertEqual(result.url, "https://files.catbox.moe/fallback.png")
manifest = read_json(asset_dir / "media.json")
self.assertEqual(selected_url(manifest["items"][0]), result.url)
manifest = _from_wire_dict(read_json(asset_dir / "media.json"))
self.assertEqual(selected_url(manifest.items[0]), result.url)
def test_note_rewrite_failure_keeps_origin_selected(self) -> None:
with tempfile.TemporaryDirectory() as raw:
@@ -54,8 +54,10 @@ class FallbackTests(unittest.TestCase):
now=NOW,
upload=lambda _path: "https://files.catbox.moe/fallback.png",
)
manifest = read_json(asset_dir / "media.json")
self.assertTrue(selected_url(manifest["items"][0]).startswith("https://pbs.twimg.com/"))
manifest = _from_wire_dict(read_json(asset_dir / "media.json"))
self.assertTrue(
(selected_url(manifest.items[0]) or "").startswith("https://pbs.twimg.com/")
)
def test_restore_origin_retains_fallback(self) -> None:
with tempfile.TemporaryDirectory() as raw:
@@ -73,10 +75,13 @@ class FallbackTests(unittest.TestCase):
upload=lambda _path: "https://files.catbox.moe/fallback.png",
)
restore_origin(asset_dir, note_dir, media_id="AAA", now=NOW)
item = read_json(asset_dir / "media.json")["items"][0]
self.assertTrue(selected_url(item).startswith("https://pbs.twimg.com/"))
self.assertTrue(any(loc.get("kind") == "fallback" for loc in item["locations"]))
manifest = _from_wire_dict(read_json(asset_dir / "media.json"))
item = manifest.items[0]
self.assertTrue(
(selected_url(item) or "").startswith("https://pbs.twimg.com/")
)
self.assertTrue(any(loc.kind == "fallback" for loc in item.locations))
if __name__ == "__main__":
unittest.main()
unittest.main()
+184 -2
View File
@@ -8,7 +8,9 @@ from twitter.media_audit import (
AuditReport,
audit_local_item,
audit_thread,
check_origin_url,
classify_origin_response,
record_origin_check,
)
from twitter.media_manifest import (
atomic_write_json,
@@ -17,6 +19,12 @@ from twitter.media_manifest import (
new_original_item,
)
from twitter.media_migrate import migrate_legacy_thread
from twitter.models import (
LegacyMediaJson,
MediaItem,
MediaManifest,
OriginCheck,
)
from twitter.tests.helpers import NOW, legacy_thread, read_json
@@ -40,8 +48,8 @@ class MediaAuditTests(unittest.TestCase):
def test_rate_limit_does_not_mean_unavailable(self) -> None:
outcome = classify_origin_response(429, "rate limited", NOW)
self.assertEqual(outcome["result"], "error")
self.assertFalse(outcome["confirms_unavailable"])
self.assertEqual(outcome.result, "error")
self.assertFalse(outcome.confirms_unavailable)
def test_frozen_thread_is_reported(self) -> None:
with tempfile.TemporaryDirectory() as raw:
@@ -53,5 +61,179 @@ class MediaAuditTests(unittest.TestCase):
self.assertTrue(any("frozen: skipped" in issue for issue in report.issues))
class OriginCheckTests(unittest.TestCase):
"""Tests for the OriginCheck boundary dataclass and its producers."""
def test_classify_available_2xx_marks_available(self) -> None:
outcome = classify_origin_response(200, "HEAD completed", NOW)
self.assertEqual(outcome.status, 200)
self.assertEqual(outcome.result, "available")
self.assertFalse(outcome.confirms_unavailable)
def test_classify_truncates_detail_to_200_chars(self) -> None:
long_detail = "x" * 500
outcome = classify_origin_response(500, long_detail, NOW)
self.assertEqual(len(outcome.detail), 200)
def test_check_origin_url_uses_opener(self) -> None:
class _FakeResponse:
status = 200
def __enter__(self) -> "_FakeResponse":
return self
def __exit__(self, *args: object) -> None:
return None
def fake_opener(*args: object, **kwargs: object) -> _FakeResponse:
return _FakeResponse()
outcome = check_origin_url(
"https://example.com/x.png",
NOW,
opener=fake_opener,
)
self.assertEqual(outcome.result, "available")
self.assertEqual(outcome.status, 200)
class RecordOriginCheckTests(unittest.TestCase):
"""Tests for the record_origin_check round-trip helper."""
def test_record_check_updates_origin_x_location(self) -> None:
item = new_original_item(
post_id="100",
media_id="AAA",
handle="example",
origin_url="https://pbs.twimg.com/media/AAA",
filename="asset.png",
local_path=Path("asset.png"),
now=NOW,
)
manifest = LegacyMediaJson.from_dict(
{
"schema_version": 2,
"root_post_id": "100",
"items": [
{
"post_id": item.post_id,
"media_id": item.media_id,
"kind": item.kind,
"role": item.role,
"handle": item.handle,
"locations": [
{
"kind": loc.kind,
"id": loc.id,
"url": loc.url,
"provider": loc.provider,
"availability": loc.availability,
}
for loc in item.locations
],
"publication": {
"selected_location_id": "origin:x",
"selected_at": NOW,
"reason": "default",
},
}
],
}
)
outcome = classify_origin_response(200, "HEAD completed", NOW)
updated = record_origin_check(manifest, 0, outcome)
updated_item: MediaItem = updated.items[0]
origin = find_location(updated_item, "origin:x")
self.assertIsNotNone(origin)
self.assertEqual(origin.checked_at, NOW)
self.assertEqual(origin.checked_status, 200)
self.assertEqual(origin.availability, "available")
self.assertIsNotNone(origin.check)
self.assertEqual(origin.check.result, "available")
def test_record_check_out_of_range_raises(self) -> None:
manifest = LegacyMediaJson.from_dict(
{"root_post_id": "100", "items": []}
)
outcome = classify_origin_response(200, "HEAD", NOW)
with self.assertRaises(ValueError):
record_origin_check(manifest, 0, outcome)
class LegacyMediaJsonTests(unittest.TestCase):
"""Tests for the LegacyMediaJson boundary dataclass."""
def test_minimal_legacy_round_trip(self) -> None:
legacy = LegacyMediaJson.from_dict(
{
"root_post_id": "100",
"items": [
{
"post_id": "100",
"media_id": "AAA",
"role": "orig",
"filename": "x.png",
"url": "https://files.catbox.moe/x.png",
}
],
}
)
self.assertEqual(legacy.root_post_id, "100")
self.assertIsNone(legacy.schema_version)
self.assertEqual(len(legacy.items), 1)
self.assertEqual(legacy.items[0].filename, "x.png")
self.assertEqual(legacy.items[0].url, "https://files.catbox.moe/x.png")
self.assertEqual(legacy.items[0].locations, ())
self.assertIsNone(legacy.mirrors)
self.assertIsNone(legacy.frozen)
def test_v2_with_mirrors_dict(self) -> None:
legacy = LegacyMediaJson.from_dict(
{
"schema_version": 2,
"root_post_id": "100",
"captured_at": NOW,
"frozen": True,
"mirrors": {
"dest1": {"destination_id": "dest1", "state": "synced"},
},
"items": [],
}
)
self.assertEqual(legacy.schema_version, 2)
self.assertEqual(legacy.captured_at, NOW)
self.assertEqual(legacy.frozen, True)
self.assertIsNotNone(legacy.mirrors)
self.assertIn("dest1", legacy.mirrors)
def test_mirrors_list_wire_is_coerced_to_dict(self) -> None:
legacy = LegacyMediaJson.from_dict(
{
"root_post_id": "100",
"items": [],
"mirrors": [
{"destination_id": "alpha", "state": "synced"},
{"destination_id": "beta", "state": "synced"},
],
}
)
self.assertIsNotNone(legacy.mirrors)
self.assertEqual(set(legacy.mirrors), {"alpha", "beta"})
def test_missing_root_post_id_raises_keyerror(self) -> None:
with self.assertRaises(KeyError):
LegacyMediaJson.from_dict({"items": []})
def test_items_not_list_raises_typeerror(self) -> None:
with self.assertRaisesRegex(TypeError, "items"):
LegacyMediaJson.from_dict({"root_post_id": "100", "items": {}})
def test_mirrors_invalid_type_raises_typeerror(self) -> None:
with self.assertRaisesRegex(TypeError, "mirrors"):
LegacyMediaJson.from_dict(
{"root_post_id": "100", "items": [], "mirrors": "nope"}
)
if __name__ == "__main__":
unittest.main()
+257 -59
View File
@@ -8,14 +8,18 @@ from pathlib import Path
from twitter.media_manifest import (
atomic_write_json,
canonical_manifest_bytes,
find_location,
hash_file,
merge_item,
merge_manifest_items,
new_original_item,
payload_inventory,
selected_url,
validate_manifest,
_from_wire_dict,
)
from twitter.models import MediaItem, MediaLocation, MediaManifest, Publication
from twitter.tests.helpers import NOW
@@ -55,9 +59,9 @@ class CanonicalManifestTests(unittest.TestCase):
local_path=local,
now=NOW,
)
self.assertEqual(selected_url(item), item["locations"][0]["url"])
self.assertEqual(find_location(item, "local")["integrity"], "present")
self.assertEqual(find_location(item, "local")["sha256"], hash_file(local))
self.assertEqual(selected_url(item), item.locations[0].url)
self.assertEqual(find_location(item, "local").integrity, "present")
self.assertEqual(find_location(item, "local").sha256, hash_file(local))
def test_missing_local_is_explicit(self) -> None:
with tempfile.TemporaryDirectory() as raw:
@@ -71,8 +75,8 @@ class CanonicalManifestTests(unittest.TestCase):
local_path=local,
now=NOW,
)
self.assertEqual(find_location(item, "local")["integrity"], "missing")
self.assertIsNone(find_location(item, "local")["sha256"])
self.assertEqual(find_location(item, "local").integrity, "missing")
self.assertIsNone(find_location(item, "local").sha256)
def test_merge_preserves_fallback_and_publication(self) -> None:
with tempfile.TemporaryDirectory() as raw:
@@ -87,24 +91,30 @@ class CanonicalManifestTests(unittest.TestCase):
local_path=local,
now=NOW,
)
old["locations"].append(
{
"id": "fallback:catbox:fixture",
"kind": "fallback",
"provider": "catbox",
"url": "https://files.catbox.moe/fallback.png",
"sha256": find_location(old, "local")["sha256"],
"uploaded_at": NOW,
"availability": "unknown",
"checked_at": None,
"check": None,
}
local_sha = find_location(old, "local").sha256
fallback = MediaLocation(
id="fallback:catbox:fixture",
kind="fallback",
provider="catbox",
url="https://files.catbox.moe/fallback.png",
sha256=local_sha,
uploaded_at=NOW,
availability="unknown",
)
old_with_fallback = MediaItem(
post_id=old.post_id,
media_id=old.media_id,
kind=old.kind,
role=old.role,
handle=old.handle,
embed=old.embed,
locations=old.locations + (fallback,),
publication=Publication(
selected_location_id="fallback:catbox:fixture",
selected_at=NOW,
reason="origin-unavailable",
),
)
old["publication"] = {
"selected_location_id": "fallback:catbox:fixture",
"selected_at": NOW,
"reason": "origin-unavailable",
}
fresh = new_original_item(
post_id="100",
media_id="AAA",
@@ -114,8 +124,10 @@ class CanonicalManifestTests(unittest.TestCase):
local_path=local,
now="2026-08-25T12:00:00Z",
)
merged = merge_item(old, fresh)
self.assertEqual(selected_url(merged), "https://files.catbox.moe/fallback.png")
merged = merge_item(old_with_fallback, fresh)
self.assertEqual(
selected_url(merged), "https://files.catbox.moe/fallback.png"
)
self.assertIsNotNone(find_location(merged, "fallback:catbox:fixture"))
self.assertEqual(merge_item(merged, fresh), merged)
@@ -145,53 +157,239 @@ class CanonicalManifestTests(unittest.TestCase):
merge_item(old, fresh)
def test_unmatched_derived_item_survives_manifest_merge(self) -> None:
derived = {
"post_id": "100",
"media_id": "AAA",
"handle": "example",
"role": "ocr",
"derived_from": {"post_id": "100", "media_id": "AAA", "role": "orig"},
"embed": False,
"locations": [],
"publication": None,
}
self.assertEqual(merge_manifest_items([derived], []), [derived])
derived = MediaItem(
post_id="100",
media_id="AAA",
kind="ocr",
role="ocr",
handle="example",
locations=(),
publication=None,
)
result = merge_manifest_items((), (derived,))
self.assertEqual(result, (derived,))
def test_validator_rejects_local_publication(self) -> None:
manifest = {
item = MediaItem(
post_id="100",
media_id="AAA",
kind="orig",
role="orig",
handle="example",
embed=True,
locations=(
MediaLocation(
id="local",
kind="local",
local_path=Path("100_AAA_orig.png"),
sha256=None,
bytes=None,
media_type="image/png",
integrity="missing",
verified_at=NOW,
),
),
publication=Publication(
selected_location_id="local",
selected_at=NOW,
reason="manual",
),
)
manifest = MediaManifest(
schema_version=2,
root_post_id="100",
items=(item,),
)
self.assertIn(
"items[0] selects non-HTTPS location local", validate_manifest(manifest)
)
class FromWireDictTests(unittest.TestCase):
"""Tests for the _from_wire_dict boundary parser."""
def test_full_wire_payload_round_trips(self) -> None:
raw: dict = {
"schema_version": 2,
"root_post_id": "100",
"captured_at": NOW,
"items": [
{
"post_id": "100",
"media_id": "AAA",
"role": "orig",
"embed": True,
"locations": [
{
"id": "origin:x",
"kind": "origin",
"provider": "x",
"url": "https://pbs.twimg.com/media/AAA",
"availability": "unknown",
"checked_at": None,
},
{
"id": "local",
"kind": "local",
"path": "100_AAA_orig.png",
"sha256": "deadbeef",
"bytes": 12345,
"media_type": "image/png",
"integrity": "present",
"verified_at": NOW,
},
],
"publication": {
"selected_location_id": "origin:x",
"selected_at": NOW,
"reason": "default",
},
}
],
"mirrors": [],
}
manifest = _from_wire_dict(raw)
self.assertEqual(manifest.schema_version, 2)
self.assertEqual(manifest.root_post_id, "100")
self.assertEqual(manifest.captured_at, NOW)
self.assertEqual(len(manifest.items), 1)
item = manifest.items[0]
self.assertEqual(item.post_id, "100")
self.assertEqual(item.media_id, "AAA")
self.assertEqual(item.kind, "orig")
self.assertEqual(item.embed, True)
self.assertEqual(len(item.locations), 2)
loc0, loc1 = item.locations
self.assertEqual(loc0.id, "origin:x")
self.assertEqual(loc0.kind, "origin")
self.assertEqual(loc0.url, "https://pbs.twimg.com/media/AAA")
self.assertEqual(loc0.provider, "x")
self.assertEqual(loc0.availability, "unknown")
self.assertEqual(loc1.id, "local")
self.assertEqual(loc1.kind, "local")
self.assertEqual(loc1.integrity, "present")
self.assertEqual(loc1.sha256, "deadbeef")
self.assertEqual(loc1.bytes, 12345)
self.assertIsNotNone(item.publication)
assert item.publication is not None
self.assertEqual(item.publication.selected_location_id, "origin:x")
self.assertEqual(item.publication.reason, "default")
def test_minimal_wire_payload(self) -> None:
raw: dict = {
"schema_version": 2,
"root_post_id": "100",
"items": [
{
"post_id": "100",
"media_id": "AAA",
"handle": "example",
"role": "orig",
"derived_from": None,
"embed": True,
"locations": [
{
"id": "local",
"kind": "local",
"path": "100_AAA_orig.png",
"sha256": None,
"bytes": None,
"media_type": "image/png",
"integrity": "missing",
"verified_at": NOW,
}
],
"publication": {
"selected_location_id": "local",
"selected_at": NOW,
"reason": "manual",
},
"locations": [],
}
],
"mirrors": [],
}
self.assertIn("items[0] selects non-HTTPS location local", validate_manifest(manifest))
manifest = _from_wire_dict(raw)
self.assertEqual(manifest.root_post_id, "100")
self.assertEqual(len(manifest.items), 1)
self.assertEqual(manifest.items[0].kind, "orig")
def test_wire_payload_with_derived_from(self) -> None:
raw: dict = {
"schema_version": 2,
"root_post_id": "100",
"items": [
{
"post_id": "100",
"media_id": "AAA",
"role": "ocr",
"derived_from": {
"post_id": "100",
"media_id": "AAA",
"role": "orig",
},
"embed": False,
"locations": [],
"publication": None,
}
],
}
manifest = _from_wire_dict(raw)
item = manifest.items[0]
self.assertEqual(item.kind, "ocr")
self.assertIsNotNone(item.derived_from)
assert item.derived_from is not None
self.assertEqual(item.derived_from.role, "orig")
self.assertIsNone(item.publication)
class CanonicalManifestBytesTests(unittest.TestCase):
"""Tests for canonical_manifest_bytes with typed MediaManifest."""
def test_canonical_manifest_bytes_is_stable(self) -> None:
manifest = MediaManifest(
schema_version=2,
root_post_id="100",
items=(
MediaItem(
post_id="100",
media_id="AAA",
kind="orig",
role="orig",
locations=(
MediaLocation(
id="origin:x",
kind="origin",
url="https://pbs.twimg.com/media/AAA",
availability="unknown",
),
),
publication=Publication(
selected_location_id="origin:x",
selected_at=NOW,
reason="default",
),
),
),
)
first = canonical_manifest_bytes(manifest)
second = canonical_manifest_bytes(manifest)
self.assertEqual(first, second)
decoded = json.loads(first.decode("utf-8"))
self.assertEqual(decoded["schema_version"], 2)
self.assertEqual(decoded["root_post_id"], "100")
self.assertEqual(decoded["mirrors"], [])
self.assertEqual(len(decoded["items"]), 1)
self.assertEqual(decoded["items"][0]["media_id"], "AAA")
def test_canonical_manifest_bytes_omits_mirrors_and_canonicalizes(self) -> None:
manifest = MediaManifest(
schema_version=2,
root_post_id="100",
items=(
MediaItem(
post_id="100",
media_id="AAA",
kind="orig",
role="orig",
locations=(
MediaLocation(
id="origin:x",
kind="origin",
url="https://x",
availability="unknown",
),
),
publication=Publication(
selected_location_id="origin:x",
selected_at=NOW,
reason="default",
),
),
),
)
encoded = canonical_manifest_bytes(manifest)
self.assertIn(b'"mirrors":[]', encoded)
if __name__ == "__main__":
unittest.main()
unittest.main()
+10 -10
View File
@@ -4,8 +4,9 @@ import tempfile
import unittest
from pathlib import Path
from twitter.media_manifest import find_location, selected_url, validate_manifest
from twitter.media_manifest import _from_wire_dict, find_location, selected_url, validate_manifest
from twitter.media_migrate import migrate_legacy_thread
from twitter.models import MediaItem
from twitter.tests.helpers import NOW, legacy_thread, read_json
@@ -22,20 +23,19 @@ class LegacyMigrationTests(unittest.TestCase):
applied = migrate_legacy_thread(asset_dir, note_dir, now=NOW, apply=True)
self.assertTrue(applied.changed)
manifest = read_json(asset_dir / "media.json")
manifest_dict = read_json(asset_dir / "media.json")
manifest = _from_wire_dict(manifest_dict)
self.assertEqual(validate_manifest(manifest), [])
item = manifest["items"][0]
item: MediaItem = manifest.items[0]
self.assertEqual(
find_location(item, "origin:x")["url"],
find_location(item, "origin:x").url,
"https://pbs.twimg.com/media/AAA?format=png&name=orig",
)
fallback = next(
location
for location in item["locations"]
if location.get("kind") == "fallback"
location for location in item.locations if location.kind == "fallback"
)
self.assertEqual(fallback["url"], "https://files.catbox.moe/fallback.png")
self.assertEqual(selected_url(item), find_location(item, "origin:x")["url"])
self.assertEqual(fallback.url, "https://files.catbox.moe/fallback.png")
self.assertEqual(selected_url(item), find_location(item, "origin:x").url)
def test_second_migration_is_idempotent(self) -> None:
with tempfile.TemporaryDirectory() as raw:
@@ -49,4 +49,4 @@ class LegacyMigrationTests(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
unittest.main()