diff --git a/archive/threads/How to archive a thread.md b/archive/threads/How to archive a thread.md index 57abdb8..ec9829a 100644 --- a/archive/threads/How to archive a thread.md +++ b/archive/threads/How to archive a thread.md @@ -20,9 +20,6 @@ aliases: in_reply_to: "" --- -> [!info] Provenance -> Stub for the note shape. A real thread is a folder: `archive/threads///index.md` plus optional branch notes. See [[How threads are organized]]. - ## Source - URL: diff --git a/archive/threads/index.md b/archive/threads/index.md index 3122857..20b2947 100644 --- a/archive/threads/index.md +++ b/archive/threads/index.md @@ -15,3 +15,5 @@ How to read a folder: [[How threads are organized]]. Example of the note shape (single spine, no branches): [[How to archive a thread]] - [[NOTimothyLottes]] +- [[VPCOMPRESSB]] +- [[winning_tactic]] diff --git a/scripts/twitter/README.md b/scripts/twitter/README.md index 3553da7..8857951 100644 --- a/scripts/twitter/README.md +++ b/scripts/twitter/README.md @@ -10,4 +10,12 @@ python C:\projects\Threadwell\scripts\twitter\emit_archive.py --input C:\project `--input` is the dump directory. `--vault` is the Threadwell root. `--slug` overrides the directory slug. `--archived` is `YYYY-MM-DD` and defaults to today. +## All dumps + +``` +python C:\projects\Threadwell\scripts\twitter\emit_archive.py --all C:\projects\manual_slop\docs\twitter --vault C:\projects\Threadwell --archived 2026-08-23 +``` + +Skips a dump whose `post_id` already exists in the vault unless `--force`. No fetch, no lift. + The converter writes the spine and branch notes under `archive/threads///` and a working copy under `assets/threads///` (`thread_data.json`, `media.json`, `gaps.md`, and any local media). Media is not published. diff --git a/scripts/twitter/emit_archive.py b/scripts/twitter/emit_archive.py index 5a864b0..41aacd1 100644 --- a/scripts/twitter/emit_archive.py +++ b/scripts/twitter/emit_archive.py @@ -4,8 +4,10 @@ from __future__ import annotations import argparse import json +import re import shutil import sys +from dataclasses import dataclass from datetime import date from pathlib import Path from urllib.parse import parse_qs, unquote, urlparse @@ -17,6 +19,17 @@ from twitter.render import render_branch, render_spine from twitter.slug import branch_file_name, date_prefix, thread_dir_name from twitter.tree import by_id, branch_roots, children_map, descendants, spine_ids +_POST_ID_LINE = re.compile(r'^post_id:\s*"?([^"\s]+)"?\s*$') +_WIKILINK_ITEM = re.compile(r"^- \[\[([^\]|]+)\]\]\s*$") + + +@dataclass(frozen=True) +class EmitResult: + post_id: str + handle: str + dir_name: str + branch_count: int + def _url_media_id(url: str, index: int) -> str: segment = Path(unquote(urlparse(url).path)).name @@ -95,6 +108,23 @@ def collect_media( return items, media_by_post +def missing_parent_ids(thread: ThreadData) -> list[str]: + ids = by_id(thread) + found: list[str] = [] + seen: set[str] = set() + for post in thread.posts: + parent = post.reply_to_id + if parent is None or parent in ids or parent in seen: + continue + seen.add(parent) + found.append(parent) + return found + + +def empty_text_ids(thread: ThreadData) -> list[str]: + return [p.post_id for p in thread.posts if not (p.text or "").strip()] + + def render_gaps(thread: ThreadData, spine: list[str]) -> str: ids = by_id(thread) suggested = spine[-1] if spine else "" @@ -104,7 +134,7 @@ def render_gaps(thread: ThreadData, spine: list[str]) -> str: for p in thread.posts if p.reply_to_id is not None and p.reply_to_id not in ids ] - empty = [p.post_id for p in thread.posts if not (p.text or "").strip()] + empty = empty_text_ids(thread) lines = [ "# gaps", @@ -123,12 +153,191 @@ def render_gaps(thread: ThreadData, spine: list[str]) -> str: return "\n".join(lines) + "\n" +def _frontmatter_block(text: str) -> str: + if not text.startswith("---"): + return "" + rest = text[3:] + if rest.startswith("\n"): + rest = rest[1:] + end = rest.find("\n---") + if end < 0: + return "" + return rest[:end] + + +def _frontmatter_post_id(text: str) -> str | None: + for line in _frontmatter_block(text).splitlines(): + match = _POST_ID_LINE.match(line.strip()) + if match: + return match.group(1) + return None + + +def collect_existing_ids(vault: Path) -> tuple[set[str], dict[str, Path]]: + """Return every archive post_id and spine dirs keyed by spine post_id.""" + threads = vault / "archive" / "threads" + ids: set[str] = set() + spines: dict[str, Path] = {} + if not threads.is_dir(): + return ids, spines + for md in threads.rglob("*.md"): + pid = _frontmatter_post_id(md.read_text(encoding="utf-8")) + if not pid: + continue + ids.add(pid) + if md.name == "index.md" and md.parent.parent.parent == threads: + spines[pid] = md.parent + return ids, spines + + +def unique_dir_name(parent: Path, base: str) -> str: + if not (parent / base).exists(): + return base + n = 2 + while (parent / f"{base}-{n}").exists(): + n += 1 + return f"{base}-{n}" + + +def _split_frontmatter(text: str) -> tuple[str, str]: + if not text.startswith("---"): + return "", text + end = text.find("\n---", 3) + if end < 0: + return "", text + closer = end + 4 + fm = text[:closer] + body = text[closer:] + if body.startswith("\n"): + fm += "\n" + body = body[1:] + elif not fm.endswith("\n"): + fm += "\n" + return fm, body + + +def _upsert_wikilink(path: Path, target: str) -> None: + text = path.read_text(encoding="utf-8") + fm, body = _split_frontmatter(text) + kept: list[str] = [] + names: list[str] = [] + for line in body.splitlines(): + match = _WIKILINK_ITEM.match(line) + if match: + name = match.group(1) + if name not in names: + names.append(name) + else: + kept.append(line) + if target not in names: + names.append(target) + names.sort() + while kept and kept[0] == "": + kept.pop(0) + while kept and kept[-1] == "": + kept.pop() + parts: list[str] = [] + if kept: + parts.extend(kept) + parts.append("") + parts.extend(f"- [[{name}]]" for name in names) + parts.append("") + body_text = "\n".join(parts) + if fm: + path.write_text(fm.rstrip() + "\n\n" + body_text, encoding="utf-8", newline="\n") + else: + path.write_text(body_text, encoding="utf-8", newline="\n") + + +def ensure_handle_index(vault: Path, handle: str, dir_name: str) -> None: + path = vault / "archive" / "threads" / handle / "index.md" + if not path.is_file(): + front = ( + "---\n" + f"title: {handle}\n" + "type: note\n" + "status: published\n" + "draft: false\n" + f"description: Archived threads by {handle}.\n" + "tags:\n" + " - archive\n" + " - twitter\n" + f" - {handle}\n" + "---\n" + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + front + f"\n- [[{dir_name}]]\n", + encoding="utf-8", + newline="\n", + ) + return + _upsert_wikilink(path, dir_name) + + +def ensure_threads_index(vault: Path, handle: str) -> None: + path = vault / "archive" / "threads" / "index.md" + if not path.is_file(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + ( + "---\n" + "title: Threads\n" + "type: note\n" + "status: published\n" + "draft: false\n" + "description: Archived Twitter/X threads and similar sequential posts.\n" + "tags:\n" + " - archive\n" + "---\n" + "\n" + "Threads are filed by author handle, then by date and title. " + "The main note in each folder is that author's own chain. " + "Replies live beside it.\n" + "\n" + "How to read a folder: [[How threads are organized]].\n" + "\n" + "Example of the note shape (single spine, no branches): " + "[[How to archive a thread]]\n" + "\n" + f"- [[{handle}]]\n" + ), + encoding="utf-8", + newline="\n", + ) + return + _upsert_wikilink(path, handle) + + +def discover_dumps(root: Path) -> list[Path]: + dumps = [ + path for path in root.iterdir() + if path.is_dir() and (path / "thread_data.json").is_file() + ] + dumps.sort(key=lambda path: path.name) + return dumps + + +def _gap_line( + post_id: str, + handle: str, + missing: list[str], + empty: list[str], +) -> str: + miss = ",".join(missing) if missing else "-" + empt = ",".join(empty) if empty else "-" + return f"{post_id} {handle} root {miss} {empt}" + + def emit( input_dir: Path, vault: Path, slug: str | None, archived: str, -) -> None: + *, + force: bool = False, + reuse_dir: Path | None = None, +) -> EmitResult: src = input_dir / "thread_data.json" if not src.is_file(): raise SystemExit(f"missing {src}") @@ -144,7 +353,25 @@ def emit( first = ids[spine[0]] handle = first.handle first_line = (first.text or "").split("\n", 1)[0] - dir_name = thread_dir_name(date_prefix(first.timestamp), first_line, slug) + + if reuse_dir is None and force: + _existing_ids, spines = collect_existing_ids(vault) + reuse_dir = spines.get(thread.root_post_id) + + if reuse_dir is not None: + dir_name = reuse_dir.name + handle = reuse_dir.parent.name + note_dir = reuse_dir + asset_dir = vault / "assets" / "threads" / handle / dir_name + else: + dir_name = thread_dir_name(date_prefix(first.timestamp), first_line, slug) + if slug is None: + dir_name = unique_dir_name( + vault / "archive" / "threads" / handle, + dir_name, + ) + note_dir = vault / "archive" / "threads" / handle / dir_name + asset_dir = vault / "assets" / "threads" / handle / dir_name branch_names: dict[str, str] = {} for rid in roots: @@ -157,8 +384,6 @@ def emit( name = fname[:-3] if fname.endswith(".md") else fname branch_names[rid] = name - note_dir = vault / "archive" / "threads" / handle / dir_name - asset_dir = vault / "assets" / "threads" / handle / dir_name note_dir.mkdir(parents=True, exist_ok=True) asset_dir.mkdir(parents=True, exist_ok=True) @@ -182,7 +407,7 @@ def emit( thread, rid, descendants(rid, kids), - handle, + first.handle, archived, media_by_post, ), @@ -206,6 +431,74 @@ def emit( newline="\n", ) + ensure_handle_index(vault, handle, dir_name) + ensure_threads_index(vault, handle) + + return EmitResult( + post_id=thread.root_post_id, + handle=handle, + dir_name=dir_name, + branch_count=len(roots), + ) + + +def emit_all( + dumps_root: Path, + vault: Path, + archived: str, + force: bool, +) -> tuple[int, int, int, int]: + dumps = discover_dumps(dumps_root) + existing_ids, spines = collect_existing_ids(vault) + lines: list[str] = [] + skipped = 0 + written = 0 + errors = 0 + + for dump in dumps: + src = dump / "thread_data.json" + try: + thread = load_thread(src) + if not thread.posts: + raise ValueError(f"no posts in {src}") + spine = spine_ids(thread) + handle = by_id(thread)[spine[0]].handle + missing = missing_parent_ids(thread) + empty = empty_text_ids(thread) + post_id = thread.root_post_id + except Exception: + lines.append(f"{dump.name} ? error - -") + errors += 1 + continue + + lines.append(_gap_line(post_id, handle, missing, empty)) + + if post_id in existing_ids and not force: + skipped += 1 + continue + + reuse = spines.get(post_id) if force else None + try: + emit( + dump, + vault, + None, + archived, + force=force, + reuse_dir=reuse, + ) + except Exception: + errors += 1 + continue + + written += 1 + existing_ids.add(post_id) + + out = vault / "assets" / "threads" / "_batch_gaps.md" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n") + return len(dumps), skipped, written, errors + def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( @@ -214,12 +507,19 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "No fetch, no lift." ), ) - parser.add_argument( + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument( "--input", - required=True, type=Path, help="Dump directory containing thread_data.json", ) + source.add_argument( + "--all", + dest="all_root", + type=Path, + metavar="DUMPS_ROOT", + help="Convert every dump directory under this root", + ) parser.add_argument( "--vault", required=True, @@ -229,19 +529,36 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--slug", default=None, - help="Optional slug override for the thread directory", + help="Optional slug override for a single --input (ignored with --all)", ) parser.add_argument( "--archived", default=date.today().isoformat(), help="Archive date YYYY-MM-DD (default: today local)", ) + parser.add_argument( + "--force", + action="store_true", + help="Emit even if post_id already exists in the vault", + ) return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: args = parse_args(argv) - emit(args.input, args.vault, args.slug, args.archived) + if args.all_root is not None: + seen, skipped, written, errors = emit_all( + args.all_root, + args.vault, + args.archived, + args.force, + ) + print( + f"dumps seen={seen} skipped={skipped} " + f"written={written} errors={errors}" + ) + return 0 if errors == 0 else 1 + emit(args.input, args.vault, args.slug, args.archived, force=args.force) return 0 diff --git a/scripts/twitter/render.py b/scripts/twitter/render.py index 0cab7f0..69f2c74 100644 --- a/scripts/twitter/render.py +++ b/scripts/twitter/render.py @@ -6,12 +6,6 @@ from .models import PostData, ThreadData from .slug import date_prefix from .tree import by_id -_PROVENANCE = ( - "> [!info] Provenance\n" - "> Primary source record. Do not editorialize here. " - "Interpretation goes in a `notes/` page." -) - def _yaml_quote(value: str) -> str: return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' @@ -149,8 +143,6 @@ def render_spine( in_reply_to=in_reply_to, ), "", - _PROVENANCE, - "", _source_block(first), "", ] @@ -200,8 +192,6 @@ def render_branch( parent_post_id=root.reply_to_id or "", ), "", - _PROVENANCE, - "", _source_block(root), "", "## Branch", diff --git a/templates/archive-source.md b/templates/archive-source.md index a0b0a9a..9201859 100644 --- a/templates/archive-source.md +++ b/templates/archive-source.md @@ -19,9 +19,6 @@ in_reply_to: "" parent_post_id: "" --- -> [!info] Provenance -> Primary source record. Do not editorialize here. Interpretation goes in a `notes/` page. - ## Source - URL: <% tp.file.cursor() %>