drafting up scripts

scripts/twitter/README.md

@@ -0,0 +1,13 @@
+# twitter
+
+Emit Threadwell archive notes from a `thread_data.json` dump. No fetch, no lift.
+
+## One dump
+
+```
+python C:\projects\Threadwell\scripts\twitter\emit_archive.py --input C:\projects\manual_slop\docs\twitter\2064858927829745887 --vault C:\projects\Threadwell --slug steamos-init --archived 2026-08-23
+```
+
+`--input` is the dump directory. `--vault` is the Threadwell root. `--slug` overrides the directory slug. `--archived` is `YYYY-MM-DD` and defaults to today.
+
+The converter writes the spine and branch notes under `archive/threads/<handle>/<date-slug>/` and a working copy under `assets/threads/<handle>/<date-slug>/` (`thread_data.json`, `media.json`, `gaps.md`, and any local media). Media is not published.

scripts/twitter/__init__.py

@@ -0,0 +1 @@
+"""Emit Threadwell archive notes from a thread_data.json dump. No fetch, no lift."""

scripts/twitter/emit_archive.py

@@ -0,0 +1,249 @@
+"""Emit Threadwell archive notes from a thread_data.json dump. No fetch, no lift."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+import sys
+from datetime import date
+from pathlib import Path
+from urllib.parse import parse_qs, unquote, urlparse
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from twitter.models import ThreadData, load_thread
+from twitter.render import render_branch, render_spine
+from twitter.slug import branch_file_name, date_prefix, thread_dir_name
+from twitter.tree import by_id, branch_roots, children_map, descendants, spine_ids
+
+
+def _url_media_id(url: str, index: int) -> str:
+    segment = Path(unquote(urlparse(url).path)).name
+    return segment if segment else f"m{index}"
+
+
+def _url_ext(url: str) -> str | None:
+    parsed = urlparse(url)
+    fmt = parse_qs(parsed.query).get("format", [None])[0]
+    if fmt:
+        return fmt.lstrip(".").lower()
+    suffix = Path(unquote(parsed.path)).suffix
+    if suffix:
+        return suffix.lstrip(".").lower()
+    return None
+
+
+def _media_files(media_dir: Path) -> list[Path]:
+    if not media_dir.is_dir():
+        return []
+    return sorted(p for p in media_dir.iterdir() if p.is_file())
+
+
+def _take_preferred(unused: list[Path], post_id: str, index: int) -> Path | None:
+    want = f"{post_id}_{index}"
+    for path in unused:
+        if path.stem == want:
+            return path
+    return None
+
+
+def _take_fallback(unused: list[Path], post_id: str) -> Path | None:
+    prefix = f"{post_id}_"
+    for path in unused:
+        if path.name.startswith(prefix):
+            return path
+    return None
+
+
+def collect_media(
+    thread: ThreadData,
+    input_dir: Path,
+    dest_dir: Path,
+) -> tuple[list[dict[str, object]], dict[str, tuple[str, ...]]]:
+    unused = _media_files(input_dir / "media")
+    items: list[dict[str, object]] = []
+    by_post: dict[str, list[str]] = {}
+
+    for post in thread.posts:
+        for index, url in enumerate(post.media_urls, start=1):
+            media_id = _url_media_id(url, index)
+            src = _take_preferred(unused, post.post_id, index)
+            if src is None:
+                src = _take_fallback(unused, post.post_id)
+            if src is not None:
+                unused.remove(src)
+                ext = src.suffix.lstrip(".") or _url_ext(url) or "bin"
+            else:
+                ext = _url_ext(url) or "bin"
+            filename = f"{post.post_id}_{media_id}_orig.{ext}"
+            if src is not None:
+                dest_dir.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(src, dest_dir / filename)
+            items.append({
+                "post_id": post.post_id,
+                "media_id": media_id,
+                "handle": post.handle,
+                "role": "orig",
+                "filename": filename,
+                "publish": False,
+                "url": None,
+            })
+            by_post.setdefault(post.post_id, []).append(filename)
+
+    media_by_post = {pid: tuple(names) for pid, names in by_post.items()}
+    return items, media_by_post
+
+
+def render_gaps(thread: ThreadData, spine: list[str]) -> str:
+    ids = by_id(thread)
+    suggested = spine[-1] if spine else ""
+    quote_ids = [p.post_id for p in thread.posts if p.quote_of_id]
+    missing = [
+        p.post_id
+        for p in thread.posts
+        if p.reply_to_id is not None and p.reply_to_id not in ids
+    ]
+    empty = [p.post_id for p in thread.posts if not (p.text or "").strip()]
+
+    lines = [
+        "# gaps",
+        "input: root",
+        f"suggested_tip: {suggested}",
+    ]
+    if quote_ids:
+        lines.append("quote_of:")
+        lines.extend(quote_ids)
+    else:
+        lines.append("quote_of: unset")
+    lines.append("missing_reply_to:")
+    lines.extend(missing)
+    lines.append("empty_text:")
+    lines.extend(empty)
+    return "\n".join(lines) + "\n"
+
+
+def emit(
+    input_dir: Path,
+    vault: Path,
+    slug: str | None,
+    archived: str,
+) -> None:
+    src = input_dir / "thread_data.json"
+    if not src.is_file():
+        raise SystemExit(f"missing {src}")
+
+    thread = load_thread(src)
+    if not thread.posts:
+        raise SystemExit(f"no posts in {src}")
+
+    spine = spine_ids(thread)
+    roots = branch_roots(thread, spine)
+    ids = by_id(thread)
+    kids = children_map(thread)
+    first = ids[spine[0]]
+    handle = first.handle
+    first_line = (first.text or "").split("\n", 1)[0]
+    dir_name = thread_dir_name(date_prefix(first.timestamp), first_line, slug)
+
+    branch_names: dict[str, str] = {}
+    for rid in roots:
+        branch = ids[rid]
+        fname = branch_file_name(
+            date_prefix(branch.timestamp),
+            branch.handle,
+            branch.text,
+        )
+        name = fname[:-3] if fname.endswith(".md") else fname
+        branch_names[rid] = name
+
+    note_dir = vault / "archive" / "threads" / handle / dir_name
+    asset_dir = vault / "assets" / "threads" / handle / dir_name
+    note_dir.mkdir(parents=True, exist_ok=True)
+    asset_dir.mkdir(parents=True, exist_ok=True)
+
+    items, media_by_post = collect_media(thread, input_dir, asset_dir)
+
+    (note_dir / "index.md").write_text(
+        render_spine(
+            thread,
+            spine,
+            roots,
+            branch_names,
+            archived,
+            media_by_post,
+        ),
+        encoding="utf-8",
+        newline="\n",
+    )
+    for rid in roots:
+        (note_dir / f"{branch_names[rid]}.md").write_text(
+            render_branch(
+                thread,
+                rid,
+                descendants(rid, kids),
+                handle,
+                archived,
+                media_by_post,
+            ),
+            encoding="utf-8",
+            newline="\n",
+        )
+
+    shutil.copy2(src, asset_dir / "thread_data.json")
+    media_doc = {
+        "root_post_id": thread.root_post_id,
+        "items": items,
+    }
+    (asset_dir / "media.json").write_text(
+        json.dumps(media_doc, indent=2) + "\n",
+        encoding="utf-8",
+        newline="\n",
+    )
+    (asset_dir / "gaps.md").write_text(
+        render_gaps(thread, spine),
+        encoding="utf-8",
+        newline="\n",
+    )
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Emit Threadwell archive notes from a thread_data.json dump. "
+            "No fetch, no lift."
+        ),
+    )
+    parser.add_argument(
+        "--input",
+        required=True,
+        type=Path,
+        help="Dump directory containing thread_data.json",
+    )
+    parser.add_argument(
+        "--vault",
+        required=True,
+        type=Path,
+        help="Threadwell vault root",
+    )
+    parser.add_argument(
+        "--slug",
+        default=None,
+        help="Optional slug override for the thread directory",
+    )
+    parser.add_argument(
+        "--archived",
+        default=date.today().isoformat(),
+        help="Archive date YYYY-MM-DD (default: today local)",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    emit(args.input, args.vault, args.slug, args.archived)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/twitter/models.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+import json
+
+
+@dataclass(frozen=True)
+class PostMetrics:
+    reply_count: int
+    repost_count: int
+    like_count: int
+    view_count: int | None
+
+
+@dataclass(frozen=True)
+class PostData:
+    post_id: str
+    author: str
+    handle: str
+    text: str
+    timestamp: str
+    media_urls: tuple[str, ...]
+    reply_to_id: str | None
+    quote_of_id: str | None
+    metrics: PostMetrics
+
+
+@dataclass(frozen=True)
+class ThreadData:
+    root_post_id: str
+    posts: tuple[PostData, ...]
+    source_url: str
+
+
+def _norm_handle(raw: str) -> str:
+    h = (raw or "").strip()
+    return h[1:] if h.startswith("@") else h
+
+
+def _norm_id(raw: object) -> str | None:
+    if raw is None:
+        return None
+    s = str(raw).strip()
+    return s or None
+
+
+def load_thread(path: Path) -> ThreadData:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    posts: list[PostData] = []
+    for pd in data.get("posts", []):
+        m = pd.get("metrics") or {}
+        view = m.get("view_count")
+        posts.append(PostData(
+            post_id=str(pd.get("post_id") or ""),
+            author=str(pd.get("author") or ""),
+            handle=_norm_handle(str(pd.get("handle") or "")),
+            text=str(pd.get("text") or ""),
+            timestamp=str(pd.get("timestamp") or ""),
+            media_urls=tuple(pd.get("media_urls") or []),
+            reply_to_id=_norm_id(pd.get("reply_to_id")),
+            quote_of_id=_norm_id(pd.get("quote_of_id")),
+            metrics=PostMetrics(
+                reply_count=int(m.get("reply_count") or 0),
+                repost_count=int(m.get("repost_count") or 0),
+                like_count=int(m.get("like_count") or 0),
+                view_count=int(view) if view is not None else None,
+            ),
+        ))
+    return ThreadData(
+        root_post_id=str(data.get("root_post_id") or ""),
+        posts=tuple(posts),
+        source_url=str(data.get("source_url") or ""),
+    )

scripts/twitter/render.py

@@ -0,0 +1,217 @@
+"""Render archive note markdown from thread JSON. Strings only; no file I/O."""
+
+from __future__ import annotations
+
+from .models import PostData, ThreadData
+from .slug import date_prefix
+from .tree import by_id
+
+_PROVENANCE = (
+    "> [!info] Provenance\n"
+    "> Primary source record. Do not editorialize here. "
+    "Interpretation goes in a `notes/` page."
+)
+
+
+def _yaml_quote(value: str) -> str:
+    return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+def _first_line(text: str) -> str:
+    return (text or "").split("\n", 1)[0].rstrip("\r")
+
+
+def _title_text(text: str) -> str:
+    line = _first_line(text)
+    idx = line.find(". ")
+    if idx >= 0:
+        return line[: idx + 1]
+    return line
+
+
+def _status_url(handle: str, post_id: str) -> str:
+    return f"https://x.com/{handle}/status/{post_id}"
+
+
+def _blockquote(text: str) -> str:
+    lines = (text or "").split("\n")
+    return "\n".join(">" if line == "" else f"> {line}" for line in lines)
+
+
+def _frontmatter(
+    *,
+    title: str,
+    source_url: str,
+    author: str,
+    handle: str,
+    post_id: str,
+    date: str,
+    archived: str,
+    spine_handle: str,
+    in_reply_to: str,
+    parent_post_id: str | None = None,
+) -> str:
+    lines = [
+        "---",
+        f"title: {_yaml_quote(title)}",
+        "type: archive",
+        "source: twitter",
+        f"source_url: {_yaml_quote(source_url)}",
+        f"author: {_yaml_quote(author)}",
+        f"handle: {handle}",
+        f"post_id: {_yaml_quote(post_id)}",
+        f"date: {date}",
+        f"archived: {archived}",
+        "status: draft",
+        "draft: true",
+        "tags:",
+        "  - archive",
+        "  - twitter",
+        f"  - {spine_handle}",
+        f"description: {_yaml_quote(title)}",
+        f"in_reply_to: {_yaml_quote(in_reply_to)}",
+    ]
+    if parent_post_id is not None:
+        lines.append(f"parent_post_id: {_yaml_quote(parent_post_id)}")
+    lines.append("---")
+    return "\n".join(lines)
+
+
+def _source_block(post: PostData) -> str:
+    url = _status_url(post.handle, post.post_id)
+    return (
+        "## Source\n"
+        "\n"
+        f"- URL: {url}\n"
+        f"- Author: {post.author} (@{post.handle})\n"
+        f"- Posted: {post.timestamp}"
+    )
+
+
+def _post_block(
+    n: int,
+    post: PostData,
+    media_by_post: dict[str, tuple[str, ...]] | None,
+    branch_links: list[str] | None = None,
+) -> str:
+    parts = [f"**{n}/**", "", post.text]
+    media = (media_by_post or {}).get(post.post_id) or ()
+    if media:
+        parts.append("")
+        parts.append("Media (not lifted): " + " ".join(f"`{fn}`" for fn in media))
+    if branch_links:
+        parts.append("")
+        parts.append("Branches: " + ", ".join(f"[[{name}]]" for name in branch_links))
+    parts.append("")
+    return "\n".join(parts)
+
+
+def _foreign_parent(
+    first: PostData,
+    ids: dict[str, PostData],
+) -> tuple[str, PostData | None]:
+    parent_id = first.reply_to_id or ""
+    if not parent_id:
+        return "", None
+    parent = ids.get(parent_id)
+    if parent is None:
+        return parent_id, None
+    if parent.handle != first.handle:
+        return parent_id, parent
+    return "", None
+
+
+def render_spine(
+    thread: ThreadData,
+    spine: list[str],
+    branch_root_ids: list[str],
+    branch_names: dict[str, str],
+    archived: str,
+    media_by_post: dict[str, tuple[str, ...]] | None = None,
+) -> str:
+    """branch_names maps branch-root post_id -> wikilink target (filename without .md)."""
+    ids = by_id(thread)
+    first = ids[spine[0]]
+    spine_handle = first.handle
+    in_reply_to, parent = _foreign_parent(first, ids)
+    title = _title_text(first.text)
+    source_url = _status_url(first.handle, first.post_id)
+    chunks = [
+        _frontmatter(
+            title=title,
+            source_url=source_url,
+            author=first.author,
+            handle=first.handle,
+            post_id=first.post_id,
+            date=date_prefix(first.timestamp),
+            archived=archived,
+            spine_handle=spine_handle,
+            in_reply_to=in_reply_to,
+        ),
+        "",
+        _PROVENANCE,
+        "",
+        _source_block(first),
+        "",
+    ]
+    if parent is not None:
+        chunks.append(_blockquote(parent.text))
+        chunks.append("")
+    chunks.append("## Thread")
+    chunks.append("")
+
+    branches_under: dict[str, list[str]] = {}
+    for rid in branch_root_ids:
+        parent_id = ids[rid].reply_to_id
+        if parent_id:
+            branches_under.setdefault(parent_id, []).append(branch_names[rid])
+
+    for n, pid in enumerate(spine, 1):
+        post = ids[pid]
+        chunks.append(_post_block(n, post, media_by_post, branches_under.get(pid)))
+
+    return "\n".join(chunks).rstrip() + "\n"
+
+
+def render_branch(
+    thread: ThreadData,
+    branch_root_id: str,
+    descendant_ids: list[str],
+    spine_handle: str,
+    archived: str,
+    media_by_post: dict[str, tuple[str, ...]] | None = None,
+) -> str:
+    """descendant_ids are posts after the root, in display order (tree.descendants)."""
+    ids = by_id(thread)
+    root = ids[branch_root_id]
+    title = _title_text(root.text)
+    source_url = _status_url(root.handle, root.post_id)
+    chunks = [
+        _frontmatter(
+            title=title,
+            source_url=source_url,
+            author=root.author,
+            handle=root.handle,
+            post_id=root.post_id,
+            date=date_prefix(root.timestamp),
+            archived=archived,
+            spine_handle=spine_handle,
+            in_reply_to="",
+            parent_post_id=root.reply_to_id or "",
+        ),
+        "",
+        _PROVENANCE,
+        "",
+        _source_block(root),
+        "",
+        "## Branch",
+        "",
+    ]
+    posts = [root] + [ids[i] for i in descendant_ids]
+    for n, post in enumerate(posts, 1):
+        chunks.append(_post_block(n, post, media_by_post))
+    chunks.append("## Related")
+    chunks.append("")
+    chunks.append("- Spine: [[index]]")
+    chunks.append("")
+    return "\n".join(chunks).rstrip() + "\n"

scripts/twitter/slug.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+import re
+
+_MENTION = re.compile(r"^(@[A-Za-z0-9_]+(\s+|$))+")
+_APOS = re.compile(r"['\u2019\u2018]")
+_NON_SLUG = re.compile(r"[^a-z0-9]+")
+
+def date_prefix(timestamp: str) -> str:
+    if len(timestamp) >= 10 and timestamp[4] == "-" and timestamp[7] == "-":
+        return timestamp[:10]
+    return "unknown-date"
+
+def slugify(text: str, max_len: int = 50) -> str:
+    s = _MENTION.sub("", (text or "").strip()).lower()
+    s = _APOS.sub("", s)
+    s = _NON_SLUG.sub("-", s).strip("-")
+    if not s:
+        return "thread"
+    if len(s) <= max_len:
+        return s
+    cut = s[:max_len]
+    if cut.endswith("-"):
+        return cut.rstrip("-")
+    cut = cut.rstrip("-")
+    if "-" in cut:
+        cut = cut.rsplit("-", 1)[0]
+    return cut or s[:max_len]
+
+def thread_dir_name(date: str, first_spine_line: str, override: str | None) -> str:
+    slug = override.strip() if override else slugify(first_spine_line)
+    return f"{date}-{slug}"
+
+def branch_file_name(date: str, handle: str, first_line: str) -> str:
+    return f"{date}-{handle}-{slugify(first_line)}.md"

scripts/twitter/tree.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from .models import ThreadData, PostData
+
+
+def by_id(thread: ThreadData) -> dict[str, PostData]:
+    return {p.post_id: p for p in thread.posts}
+
+
+def children_map(thread: ThreadData) -> dict[str, list[str]]:
+    ids = by_id(thread)
+    kids: dict[str, list[str]] = {p.post_id: [] for p in thread.posts}
+    for p in thread.posts:
+        if p.reply_to_id and p.reply_to_id in ids:
+            kids[p.reply_to_id].append(p.post_id)
+    for parent, lst in kids.items():
+        lst.sort(key=lambda i: (ids[i].timestamp, ids[i].post_id))
+    return kids
+
+
+def spine_ids(thread: ThreadData) -> list[str]:
+    ids = by_id(thread)
+    kids = children_map(thread)
+    start = thread.root_post_id if thread.root_post_id in ids else thread.posts[0].post_id
+    handle = ids[start].handle
+    out = [start]
+    cur = start
+    while True:
+        same = [c for c in kids[cur] if ids[c].handle == handle]
+        if not same:
+            break
+        cur = same[0]
+        out.append(cur)
+    return out
+
+
+def branch_roots(thread: ThreadData, spine: list[str]) -> list[str]:
+    spine_set = set(spine)
+    ids = by_id(thread)
+    roots: list[str] = []
+    for p in thread.posts:
+        if p.post_id in spine_set:
+            continue
+        parent_on_spine = p.reply_to_id in spine_set
+        parent_missing = p.reply_to_id is not None and p.reply_to_id not in ids
+        if parent_on_spine or parent_missing:
+            roots.append(p.post_id)
+    roots.sort(key=lambda i: (ids[i].timestamp, ids[i].post_id))
+    return roots
+
+
+def descendants(start_id: str, kids: dict[str, list[str]]) -> list[str]:
+    out: list[str] = []
+    stack = list(reversed(kids.get(start_id, [])))
+    while stack:
+        n = stack.pop()
+        out.append(n)
+        stack.extend(reversed(kids.get(n, [])))
+    return out