setting up scripts, testing...

archive/threads/How to archive a thread.md

@@ -20,9 +20,6 @@ aliases:
 in_reply_to: ""
 ---
 
-> [!info] Provenance
-> Stub for the note shape. A real thread is a folder: `archive/threads/<handle>/<YYYY-MM-DD-slug>/index.md` plus optional branch notes. See [[How threads are organized]].
-
 ## Source
 
 - URL:

archive/threads/index.md

@@ -15,3 +15,5 @@ How to read a folder: [[How threads are organized]].
 Example of the note shape (single spine, no branches): [[How to archive a thread]]
 
 - [[NOTimothyLottes]]
+- [[VPCOMPRESSB]]
+- [[winning_tactic]]

scripts/twitter/README.md

@@ -10,4 +10,12 @@ python C:\projects\Threadwell\scripts\twitter\emit_archive.py --input C:\project
 
 `--input` is the dump directory. `--vault` is the Threadwell root. `--slug` overrides the directory slug. `--archived` is `YYYY-MM-DD` and defaults to today.
 
+## All dumps
+
+```
+python C:\projects\Threadwell\scripts\twitter\emit_archive.py --all C:\projects\manual_slop\docs\twitter --vault C:\projects\Threadwell --archived 2026-08-23
+```
+
+Skips a dump whose `post_id` already exists in the vault unless `--force`. No fetch, no lift.
+
 The converter writes the spine and branch notes under `archive/threads/<handle>/<date-slug>/` and a working copy under `assets/threads/<handle>/<date-slug>/` (`thread_data.json`, `media.json`, `gaps.md`, and any local media). Media is not published.

scripts/twitter/emit_archive.py

@@ -4,8 +4,10 @@ from __future__ import annotations
 
 import argparse
 import json
+import re
 import shutil
 import sys
+from dataclasses import dataclass
 from datetime import date
 from pathlib import Path
 from urllib.parse import parse_qs, unquote, urlparse
@@ -17,6 +19,17 @@ from twitter.render import render_branch, render_spine
 from twitter.slug import branch_file_name, date_prefix, thread_dir_name
 from twitter.tree import by_id, branch_roots, children_map, descendants, spine_ids
 
+_POST_ID_LINE = re.compile(r'^post_id:\s*"?([^"\s]+)"?\s*$')
+_WIKILINK_ITEM = re.compile(r"^- \[\[([^\]|]+)\]\]\s*$")
+
+
+@dataclass(frozen=True)
+class EmitResult:
+    post_id: str
+    handle: str
+    dir_name: str
+    branch_count: int
+
 
 def _url_media_id(url: str, index: int) -> str:
     segment = Path(unquote(urlparse(url).path)).name
@@ -95,6 +108,23 @@ def collect_media(
     return items, media_by_post
 
 
+def missing_parent_ids(thread: ThreadData) -> list[str]:
+    ids = by_id(thread)
+    found: list[str] = []
+    seen: set[str] = set()
+    for post in thread.posts:
+        parent = post.reply_to_id
+        if parent is None or parent in ids or parent in seen:
+            continue
+        seen.add(parent)
+        found.append(parent)
+    return found
+
+
+def empty_text_ids(thread: ThreadData) -> list[str]:
+    return [p.post_id for p in thread.posts if not (p.text or "").strip()]
+
+
 def render_gaps(thread: ThreadData, spine: list[str]) -> str:
     ids = by_id(thread)
     suggested = spine[-1] if spine else ""
@@ -104,7 +134,7 @@ def render_gaps(thread: ThreadData, spine: list[str]) -> str:
         for p in thread.posts
         if p.reply_to_id is not None and p.reply_to_id not in ids
     ]
-    empty = [p.post_id for p in thread.posts if not (p.text or "").strip()]
+    empty = empty_text_ids(thread)
 
     lines = [
         "# gaps",
@@ -123,12 +153,191 @@ def render_gaps(thread: ThreadData, spine: list[str]) -> str:
     return "\n".join(lines) + "\n"
 
 
+def _frontmatter_block(text: str) -> str:
+    if not text.startswith("---"):
+        return ""
+    rest = text[3:]
+    if rest.startswith("\n"):
+        rest = rest[1:]
+    end = rest.find("\n---")
+    if end < 0:
+        return ""
+    return rest[:end]
+
+
+def _frontmatter_post_id(text: str) -> str | None:
+    for line in _frontmatter_block(text).splitlines():
+        match = _POST_ID_LINE.match(line.strip())
+        if match:
+            return match.group(1)
+    return None
+
+
+def collect_existing_ids(vault: Path) -> tuple[set[str], dict[str, Path]]:
+    """Return every archive post_id and spine dirs keyed by spine post_id."""
+    threads = vault / "archive" / "threads"
+    ids: set[str] = set()
+    spines: dict[str, Path] = {}
+    if not threads.is_dir():
+        return ids, spines
+    for md in threads.rglob("*.md"):
+        pid = _frontmatter_post_id(md.read_text(encoding="utf-8"))
+        if not pid:
+            continue
+        ids.add(pid)
+        if md.name == "index.md" and md.parent.parent.parent == threads:
+            spines[pid] = md.parent
+    return ids, spines
+
+
+def unique_dir_name(parent: Path, base: str) -> str:
+    if not (parent / base).exists():
+        return base
+    n = 2
+    while (parent / f"{base}-{n}").exists():
+        n += 1
+    return f"{base}-{n}"
+
+
+def _split_frontmatter(text: str) -> tuple[str, str]:
+    if not text.startswith("---"):
+        return "", text
+    end = text.find("\n---", 3)
+    if end < 0:
+        return "", text
+    closer = end + 4
+    fm = text[:closer]
+    body = text[closer:]
+    if body.startswith("\n"):
+        fm += "\n"
+        body = body[1:]
+    elif not fm.endswith("\n"):
+        fm += "\n"
+    return fm, body
+
+
+def _upsert_wikilink(path: Path, target: str) -> None:
+    text = path.read_text(encoding="utf-8")
+    fm, body = _split_frontmatter(text)
+    kept: list[str] = []
+    names: list[str] = []
+    for line in body.splitlines():
+        match = _WIKILINK_ITEM.match(line)
+        if match:
+            name = match.group(1)
+            if name not in names:
+                names.append(name)
+        else:
+            kept.append(line)
+    if target not in names:
+        names.append(target)
+    names.sort()
+    while kept and kept[0] == "":
+        kept.pop(0)
+    while kept and kept[-1] == "":
+        kept.pop()
+    parts: list[str] = []
+    if kept:
+        parts.extend(kept)
+        parts.append("")
+    parts.extend(f"- [[{name}]]" for name in names)
+    parts.append("")
+    body_text = "\n".join(parts)
+    if fm:
+        path.write_text(fm.rstrip() + "\n\n" + body_text, encoding="utf-8", newline="\n")
+    else:
+        path.write_text(body_text, encoding="utf-8", newline="\n")
+
+
+def ensure_handle_index(vault: Path, handle: str, dir_name: str) -> None:
+    path = vault / "archive" / "threads" / handle / "index.md"
+    if not path.is_file():
+        front = (
+            "---\n"
+            f"title: {handle}\n"
+            "type: note\n"
+            "status: published\n"
+            "draft: false\n"
+            f"description: Archived threads by {handle}.\n"
+            "tags:\n"
+            "  - archive\n"
+            "  - twitter\n"
+            f"  - {handle}\n"
+            "---\n"
+        )
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(
+            front + f"\n- [[{dir_name}]]\n",
+            encoding="utf-8",
+            newline="\n",
+        )
+        return
+    _upsert_wikilink(path, dir_name)
+
+
+def ensure_threads_index(vault: Path, handle: str) -> None:
+    path = vault / "archive" / "threads" / "index.md"
+    if not path.is_file():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(
+            (
+                "---\n"
+                "title: Threads\n"
+                "type: note\n"
+                "status: published\n"
+                "draft: false\n"
+                "description: Archived Twitter/X threads and similar sequential posts.\n"
+                "tags:\n"
+                "  - archive\n"
+                "---\n"
+                "\n"
+                "Threads are filed by author handle, then by date and title. "
+                "The main note in each folder is that author's own chain. "
+                "Replies live beside it.\n"
+                "\n"
+                "How to read a folder: [[How threads are organized]].\n"
+                "\n"
+                "Example of the note shape (single spine, no branches): "
+                "[[How to archive a thread]]\n"
+                "\n"
+                f"- [[{handle}]]\n"
+            ),
+            encoding="utf-8",
+            newline="\n",
+        )
+        return
+    _upsert_wikilink(path, handle)
+
+
+def discover_dumps(root: Path) -> list[Path]:
+    dumps = [
+        path for path in root.iterdir()
+        if path.is_dir() and (path / "thread_data.json").is_file()
+    ]
+    dumps.sort(key=lambda path: path.name)
+    return dumps
+
+
+def _gap_line(
+    post_id: str,
+    handle: str,
+    missing: list[str],
+    empty: list[str],
+) -> str:
+    miss = ",".join(missing) if missing else "-"
+    empt = ",".join(empty) if empty else "-"
+    return f"{post_id} {handle} root {miss} {empt}"
+
+
 def emit(
     input_dir: Path,
     vault: Path,
     slug: str | None,
     archived: str,
-) -> None:
+    *,
+    force: bool = False,
+    reuse_dir: Path | None = None,
+) -> EmitResult:
     src = input_dir / "thread_data.json"
     if not src.is_file():
         raise SystemExit(f"missing {src}")
@@ -144,7 +353,25 @@ def emit(
     first = ids[spine[0]]
     handle = first.handle
     first_line = (first.text or "").split("\n", 1)[0]
-    dir_name = thread_dir_name(date_prefix(first.timestamp), first_line, slug)
+
+    if reuse_dir is None and force:
+        _existing_ids, spines = collect_existing_ids(vault)
+        reuse_dir = spines.get(thread.root_post_id)
+
+    if reuse_dir is not None:
+        dir_name = reuse_dir.name
+        handle = reuse_dir.parent.name
+        note_dir = reuse_dir
+        asset_dir = vault / "assets" / "threads" / handle / dir_name
+    else:
+        dir_name = thread_dir_name(date_prefix(first.timestamp), first_line, slug)
+        if slug is None:
+            dir_name = unique_dir_name(
+                vault / "archive" / "threads" / handle,
+                dir_name,
+            )
+        note_dir = vault / "archive" / "threads" / handle / dir_name
+        asset_dir = vault / "assets" / "threads" / handle / dir_name
 
     branch_names: dict[str, str] = {}
     for rid in roots:
@@ -157,8 +384,6 @@ def emit(
         name = fname[:-3] if fname.endswith(".md") else fname
         branch_names[rid] = name
 
-    note_dir = vault / "archive" / "threads" / handle / dir_name
-    asset_dir = vault / "assets" / "threads" / handle / dir_name
     note_dir.mkdir(parents=True, exist_ok=True)
     asset_dir.mkdir(parents=True, exist_ok=True)
 
@@ -182,7 +407,7 @@ def emit(
                 thread,
                 rid,
                 descendants(rid, kids),
-                handle,
+                first.handle,
                 archived,
                 media_by_post,
             ),
@@ -206,6 +431,74 @@ def emit(
         newline="\n",
     )
 
+    ensure_handle_index(vault, handle, dir_name)
+    ensure_threads_index(vault, handle)
+
+    return EmitResult(
+        post_id=thread.root_post_id,
+        handle=handle,
+        dir_name=dir_name,
+        branch_count=len(roots),
+    )
+
+
+def emit_all(
+    dumps_root: Path,
+    vault: Path,
+    archived: str,
+    force: bool,
+) -> tuple[int, int, int, int]:
+    dumps = discover_dumps(dumps_root)
+    existing_ids, spines = collect_existing_ids(vault)
+    lines: list[str] = []
+    skipped = 0
+    written = 0
+    errors = 0
+
+    for dump in dumps:
+        src = dump / "thread_data.json"
+        try:
+            thread = load_thread(src)
+            if not thread.posts:
+                raise ValueError(f"no posts in {src}")
+            spine = spine_ids(thread)
+            handle = by_id(thread)[spine[0]].handle
+            missing = missing_parent_ids(thread)
+            empty = empty_text_ids(thread)
+            post_id = thread.root_post_id
+        except Exception:
+            lines.append(f"{dump.name} ? error - -")
+            errors += 1
+            continue
+
+        lines.append(_gap_line(post_id, handle, missing, empty))
+
+        if post_id in existing_ids and not force:
+            skipped += 1
+            continue
+
+        reuse = spines.get(post_id) if force else None
+        try:
+            emit(
+                dump,
+                vault,
+                None,
+                archived,
+                force=force,
+                reuse_dir=reuse,
+            )
+        except Exception:
+            errors += 1
+            continue
+
+        written += 1
+        existing_ids.add(post_id)
+
+    out = vault / "assets" / "threads" / "_batch_gaps.md"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n")
+    return len(dumps), skipped, written, errors
+
 
 def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser = argparse.ArgumentParser(
@@ -214,12 +507,19 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
             "No fetch, no lift."
         ),
     )
-    parser.add_argument(
+    source = parser.add_mutually_exclusive_group(required=True)
+    source.add_argument(
         "--input",
-        required=True,
         type=Path,
         help="Dump directory containing thread_data.json",
     )
+    source.add_argument(
+        "--all",
+        dest="all_root",
+        type=Path,
+        metavar="DUMPS_ROOT",
+        help="Convert every dump directory under this root",
+    )
     parser.add_argument(
         "--vault",
         required=True,
@@ -229,19 +529,36 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser.add_argument(
         "--slug",
         default=None,
-        help="Optional slug override for the thread directory",
+        help="Optional slug override for a single --input (ignored with --all)",
     )
     parser.add_argument(
         "--archived",
         default=date.today().isoformat(),
         help="Archive date YYYY-MM-DD (default: today local)",
     )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Emit even if post_id already exists in the vault",
+    )
     return parser.parse_args(argv)
 
 
 def main(argv: list[str] | None = None) -> int:
     args = parse_args(argv)
-    emit(args.input, args.vault, args.slug, args.archived)
+    if args.all_root is not None:
+        seen, skipped, written, errors = emit_all(
+            args.all_root,
+            args.vault,
+            args.archived,
+            args.force,
+        )
+        print(
+            f"dumps seen={seen} skipped={skipped} "
+            f"written={written} errors={errors}"
+        )
+        return 0 if errors == 0 else 1
+    emit(args.input, args.vault, args.slug, args.archived, force=args.force)
     return 0
 
 

scripts/twitter/render.py

@@ -6,12 +6,6 @@ from .models import PostData, ThreadData
 from .slug import date_prefix
 from .tree import by_id
 
-_PROVENANCE = (
-    "> [!info] Provenance\n"
-    "> Primary source record. Do not editorialize here. "
-    "Interpretation goes in a `notes/` page."
-)
-
 
 def _yaml_quote(value: str) -> str:
     return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
@@ -149,8 +143,6 @@ def render_spine(
             in_reply_to=in_reply_to,
         ),
         "",
-        _PROVENANCE,
-        "",
         _source_block(first),
         "",
     ]
@@ -200,8 +192,6 @@ def render_branch(
             parent_post_id=root.reply_to_id or "",
         ),
         "",
-        _PROVENANCE,
-        "",
         _source_block(root),
         "",
         "## Branch",

templates/archive-source.md

@@ -19,9 +19,6 @@ in_reply_to: ""
 parent_post_id: ""
 ---
 
-> [!info] Provenance
-> Primary source record. Do not editorialize here. Interpretation goes in a `notes/` page.
-
 ## Source
 
 - URL: <% tp.file.cursor() %>