Private
Public Access
conductor(entropy_epiplexity): Phase 4 Synthesis - report.md (1,018 lines) + summary.md (341 words)
Deep-dive report covers all 8 sections per umbrella spec FR6: - TL;DR: epiplexity as observer-relative information measure - Key Concepts: 18 numbered concepts - Frame Analysis: 176 unique frames from research talk - Transcript Highlights: 10+ verbatim passages with timestamps - Mathematical Content: 12 derivations (Shannon, Kolmogorov, Levin, sophistication, epiplexity) - Connections: forward refs to 8 other videos - Open Questions: 14 questions for Pass 2 - References: people, concepts, resources Plus 9 appendices: concept map, transcript excerpts (C.1-C.12), math foundations (D.1-D.10), framework connections (E.1-E.7), cross-references (G.1-G.9), resources, final notes. Lossless preservation per umbrella spec §0.
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
"""Quick dedup pass for entropy_epiplexity (frames extracted but not deduped)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from PIL import Image
|
||||
import imagehash
|
||||
|
||||
|
||||
def main() -> int:
|
||||
frames_dir = ROOT / "conductor" / "tracks" / "video_analysis_entropy_epiplexity_20260621" / "artifacts" / "frames"
|
||||
frame_files = sorted(frames_dir.glob("frame_*.jpg"))
|
||||
print(f"Total frames: {len(frame_files)}")
|
||||
saved_hashes: list[str] = []
|
||||
kept_files: list[str] = []
|
||||
for fp in frame_files:
|
||||
img = Image.open(fp)
|
||||
h = str(imagehash.phash(img))
|
||||
if any(_hamming(h, s) < 5 for s in saved_hashes):
|
||||
fp.unlink()
|
||||
continue
|
||||
saved_hashes.append(h)
|
||||
kept_files.append(fp.name)
|
||||
print(f"Kept: {len(kept_files)}")
|
||||
meta = {
|
||||
"video": "video.mp4",
|
||||
"threshold": 0.05,
|
||||
"total_extracted": len(frame_files),
|
||||
"kept": len(kept_files),
|
||||
"files": kept_files,
|
||||
}
|
||||
(frames_dir / "extraction_meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
def _hamming(a: str, b: str) -> int:
|
||||
if len(a) != len(b):
|
||||
return max(len(a), len(b))
|
||||
return sum(1 for x, y in zip(a, b) if x != y)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -17,7 +17,7 @@ ROOT = Path(__file__).resolve().parents[4]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from scripts.video_analysis.download_video import download_video
|
||||
from scripts.video_analysis.extract_transcript import _fetch_raw_transcript
|
||||
from scripts.video_analysis.extract_transcript import _fetch_via_ytdlp as _fetch_raw_transcript
|
||||
|
||||
|
||||
def _parse_vtt_segments(vtt_path: Path) -> list[dict]:
|
||||
@@ -48,18 +48,18 @@ def phase1_acquire(slug: str, url: str, artifacts_dir: Path) -> dict:
|
||||
return {"status": "error", "error": f"Could not parse video_id from {url}"}
|
||||
video_id = m.group(1)
|
||||
|
||||
print("Step 1: extract_transcript (try youtube-transcript-api)")
|
||||
print("Step 1: extract_transcript (yt-dlp VTT directly)")
|
||||
transcript_path = artifacts_dir / "transcript.json"
|
||||
last_exc = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
segments = _fetch_raw_transcript(video_id)
|
||||
segments = _fetch_raw_transcript(video_id, artifacts_dir)
|
||||
data = {
|
||||
"video_id": video_id,
|
||||
"segments": segments,
|
||||
"plain": "\n".join(s["text"] for s in segments),
|
||||
"fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"source": "youtube-transcript-api",
|
||||
"source": "yt-dlp-vtt",
|
||||
}
|
||||
transcript_path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(f" OK: wrote {transcript_path} ({len(segments)} segments)")
|
||||
@@ -72,37 +72,16 @@ def phase1_acquire(slug: str, url: str, artifacts_dir: Path) -> dict:
|
||||
time.sleep(2 ** attempt)
|
||||
|
||||
if last_exc is not None:
|
||||
print("Step 1b: yt-dlp subtitle fallback")
|
||||
completed = subprocess.run(
|
||||
["yt-dlp", "--write-auto-subs", "--sub-langs", "en", "--sub-format", "vtt",
|
||||
"--skip-download", "--output", str(artifacts_dir / video_id), url],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
candidates = list(artifacts_dir.glob(f"{video_id}*.vtt"))
|
||||
if not candidates:
|
||||
print(f" yt-dlp subtitle fetch failed: {completed.stderr[:300]}")
|
||||
print(f" No transcript available. Continuing with download only.")
|
||||
transcript_path.write_text(json.dumps({
|
||||
"video_id": video_id,
|
||||
"segments": [],
|
||||
"plain": "",
|
||||
"fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"source": "unavailable",
|
||||
"error": str(last_exc)[:500],
|
||||
"note": "youtube-transcript-api and yt-dlp VTT both failed. Frame OCR will be the primary signal.",
|
||||
}, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
else:
|
||||
vtt_path = candidates[0]
|
||||
segments = _parse_vtt_segments(vtt_path)
|
||||
data = {
|
||||
"video_id": video_id,
|
||||
"segments": segments,
|
||||
"plain": "\n".join(s["text"] for s in segments),
|
||||
"fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"source": "yt-dlp-vtt",
|
||||
}
|
||||
transcript_path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
print(f" OK: parsed {len(segments)} segments from {vtt_path.name}")
|
||||
print(f" yt-dlp VTT fetch failed after 3 attempts. No transcript available.")
|
||||
transcript_path.write_text(json.dumps({
|
||||
"video_id": video_id,
|
||||
"segments": [],
|
||||
"plain": "",
|
||||
"fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"source": "unavailable",
|
||||
"error": str(last_exc)[:500],
|
||||
"note": "Frame OCR will be the primary signal for this video.",
|
||||
}, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
print("Step 2: download_video")
|
||||
video_path = artifacts_dir / "video.mp4"
|
||||
|
||||
Reference in New Issue
Block a user