From c18ee1b4b59292234c7577a095ed171b22ae6e22 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 23 Aug 2026 14:46:05 -0400 Subject: [PATCH] drafting up scripts --- .gitignore | 6 + README.md | 1 + ...o-give-artix-lxqt-runit-variation-a-try.md | 47 ++++ ...e-is-this-thin-with-win11-right-now-but.md | 39 +++ ...ing-to-me-how-this-challenge-would-have.md | 43 +++ .../2026-06-10-steamos-init/index.md | 40 +++ archive/threads/NOTimothyLottes/index.md | 13 + archive/threads/index.md | 2 + assets/README.md | 1 + meta/conventions.md | 1 + publish/quartz.config.yaml | 3 + scripts/twitter/README.md | 13 + scripts/twitter/__init__.py | 1 + scripts/twitter/emit_archive.py | 249 ++++++++++++++++++ scripts/twitter/models.py | 74 ++++++ scripts/twitter/render.py | 217 +++++++++++++++ scripts/twitter/slug.py | 34 +++ scripts/twitter/tree.py | 59 +++++ secrets/README.md | 1 + 19 files changed, 844 insertions(+) create mode 100644 archive/threads/NOTimothyLottes/2026-06-10-steamos-init/2026-06-11-NOTimothyLottes-going-to-give-artix-lxqt-runit-variation-a-try.md create mode 100644 archive/threads/NOTimothyLottes/2026-06-10-steamos-init/2026-06-11-RouaniJihad-my-patience-is-this-thin-with-win11-right-now-but.md create mode 100644 archive/threads/NOTimothyLottes/2026-06-10-steamos-init/2026-06-11-mcnabbd-fascinating-to-me-how-this-challenge-would-have.md create mode 100644 archive/threads/NOTimothyLottes/2026-06-10-steamos-init/index.md create mode 100644 archive/threads/NOTimothyLottes/index.md create mode 100644 assets/README.md create mode 100644 scripts/twitter/README.md create mode 100644 scripts/twitter/__init__.py create mode 100644 scripts/twitter/emit_archive.py create mode 100644 scripts/twitter/models.py create mode 100644 scripts/twitter/render.py create mode 100644 scripts/twitter/slug.py create mode 100644 scripts/twitter/tree.py create mode 100644 secrets/README.md diff --git a/.gitignore b/.gitignore index e52e02d..a7b6c44 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,9 @@ meta/attachments/* .obsidian/plugins/ .obsidian/themes/ .obsidian/snippets/ + +assets/* +!assets/README.md +secrets/* +!secrets/README.md +scripts/twitter/__pycache__ diff --git a/README.md b/README.md index 4a5d0b9..9d69c0d 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ templates/ # not published meta/ # not published — conventions assets/ # not published, gitignored — working media secrets/ # not published, gitignored — cookies, host keys +scripts/ # not published — local tooling (tracked) private/ # not published, not pushed publish/ # tracked overlay: quartz.config.yaml site/ # local Quartz clone (gitignored; own remotes) diff --git a/archive/threads/NOTimothyLottes/2026-06-10-steamos-init/2026-06-11-NOTimothyLottes-going-to-give-artix-lxqt-runit-variation-a-try.md b/archive/threads/NOTimothyLottes/2026-06-10-steamos-init/2026-06-11-NOTimothyLottes-going-to-give-artix-lxqt-runit-variation-a-try.md new file mode 100644 index 0000000..0fde2b2 --- /dev/null +++ b/archive/threads/NOTimothyLottes/2026-06-10-steamos-init/2026-06-11-NOTimothyLottes-going-to-give-artix-lxqt-runit-variation-a-try.md @@ -0,0 +1,47 @@ +--- +title: "Going to give artix-lxqt-runit variation a try." +type: archive +source: twitter +source_url: "https://x.com/NOTimothyLottes/status/2064877857252291055" +author: "NOTimothyLottes" +handle: NOTimothyLottes +post_id: "2064877857252291055" +date: 2026-06-11 +archived: 2026-08-23 +status: draft +draft: true +tags: + - archive + - twitter + - NOTimothyLottes +description: "Going to give artix-lxqt-runit variation a try." +in_reply_to: "" +parent_post_id: "2064858927829745887" +--- + +> [!info] Provenance +> Primary source record. Do not editorialize here. Interpretation goes in a `notes/` page. + +## Source + +- URL: https://x.com/NOTimothyLottes/status/2064877857252291055 +- Author: NOTimothyLottes (@NOTimothyLottes) +- Posted: 2026-06-11 01:10:19 + +## Branch + +**1/** + +Going to give artix-lxqt-runit variation a try. Mostly just a stop gap to see if I can actually get a bloated Linux system installed before a more serious effort with BuildRoot. First de-windows victim computer will be an AMD powered APU. + +**2/** + +Looks like the workaround for my locked UEFI secure boot Enabled on the second machine was to "Set Supervisor Password", afterwords I could disable secure boot and APM, but still no way to enable legacy BIOS "Boot Mode". + +**3/** + +Somewhat surprised, Artix using the graphical install just worked out of the box (AMD APU). Got confused post install when 'top' shows 'Xorg' instead of 'XLibre' but the package manager says XLibre is installed. + +## Related + +- Spine: [[index]] diff --git a/archive/threads/NOTimothyLottes/2026-06-10-steamos-init/2026-06-11-RouaniJihad-my-patience-is-this-thin-with-win11-right-now-but.md b/archive/threads/NOTimothyLottes/2026-06-10-steamos-init/2026-06-11-RouaniJihad-my-patience-is-this-thin-with-win11-right-now-but.md new file mode 100644 index 0000000..cddc0da --- /dev/null +++ b/archive/threads/NOTimothyLottes/2026-06-10-steamos-init/2026-06-11-RouaniJihad-my-patience-is-this-thin-with-win11-right-now-but.md @@ -0,0 +1,39 @@ +--- +title: "@NOTimothyLottes My patience is this 🤏 thin with win11 right now, but Linux sounds like too much work, Mac is probably easier with more native software" +type: archive +source: twitter +source_url: "https://x.com/RouaniJihad/status/2064865782870294858" +author: "🕹️G🅰MESTUFFS🎮" +handle: RouaniJihad +post_id: "2064865782870294858" +date: 2026-06-11 +archived: 2026-08-23 +status: draft +draft: true +tags: + - archive + - twitter + - NOTimothyLottes +description: "@NOTimothyLottes My patience is this 🤏 thin with win11 right now, but Linux sounds like too much work, Mac is probably easier with more native software" +in_reply_to: "" +parent_post_id: "2064858927829745887" +--- + +> [!info] Provenance +> Primary source record. Do not editorialize here. Interpretation goes in a `notes/` page. + +## Source + +- URL: https://x.com/RouaniJihad/status/2064865782870294858 +- Author: 🕹️G🅰MESTUFFS🎮 (@RouaniJihad) +- Posted: 2026-06-11 00:22:20 + +## Branch + +**1/** + +@NOTimothyLottes My patience is this 🤏 thin with win11 right now, but Linux sounds like too much work, Mac is probably easier with more native software + +## Related + +- Spine: [[index]] diff --git a/archive/threads/NOTimothyLottes/2026-06-10-steamos-init/2026-06-11-mcnabbd-fascinating-to-me-how-this-challenge-would-have.md b/archive/threads/NOTimothyLottes/2026-06-10-steamos-init/2026-06-11-mcnabbd-fascinating-to-me-how-this-challenge-would-have.md new file mode 100644 index 0000000..6792aa2 --- /dev/null +++ b/archive/threads/NOTimothyLottes/2026-06-10-steamos-init/2026-06-11-mcnabbd-fascinating-to-me-how-this-challenge-would-have.md @@ -0,0 +1,43 @@ +--- +title: "@NOTimothyLottes fascinating to me how this challenge would have captivated me a couple decades ago but now i just wanna run away" +type: archive +source: twitter +source_url: "https://x.com/mcnabbd/status/2064875327617396915" +author: "Doug McNabb" +handle: mcnabbd +post_id: "2064875327617396915" +date: 2026-06-11 +archived: 2026-08-23 +status: draft +draft: true +tags: + - archive + - twitter + - NOTimothyLottes +description: "@NOTimothyLottes fascinating to me how this challenge would have captivated me a couple decades ago but now i just wanna run away" +in_reply_to: "" +parent_post_id: "2064858927829745887" +--- + +> [!info] Provenance +> Primary source record. Do not editorialize here. Interpretation goes in a `notes/` page. + +## Source + +- URL: https://x.com/mcnabbd/status/2064875327617396915 +- Author: Doug McNabb (@mcnabbd) +- Posted: 2026-06-11 01:00:16 + +## Branch + +**1/** + +@NOTimothyLottes fascinating to me how this challenge would have captivated me a couple decades ago but now i just wanna run away + +**2/** + +@mcnabbd Similar feelings. Used to build linux from scratch in the BIOS era, also ran FreeBSD/etc, just for fun. But things had been easier then. Now years of technical debt later, all my tolerance for complexity has been fully exhausted. + +## Related + +- Spine: [[index]] diff --git a/archive/threads/NOTimothyLottes/2026-06-10-steamos-init/index.md b/archive/threads/NOTimothyLottes/2026-06-10-steamos-init/index.md new file mode 100644 index 0000000..0b4644c --- /dev/null +++ b/archive/threads/NOTimothyLottes/2026-06-10-steamos-init/index.md @@ -0,0 +1,40 @@ +--- +title: "So I'm on Linux now, but obviously I cheated, because this is a SteamOS machine (pre-installed)." +type: archive +source: twitter +source_url: "https://x.com/NOTimothyLottes/status/2064858927829745887" +author: "NOTimothyLottes" +handle: NOTimothyLottes +post_id: "2064858927829745887" +date: 2026-06-10 +archived: 2026-08-23 +status: draft +draft: true +tags: + - archive + - twitter + - NOTimothyLottes +description: "So I'm on Linux now, but obviously I cheated, because this is a SteamOS machine (pre-installed)." +in_reply_to: "" +--- + +> [!info] Provenance +> Primary source record. Do not editorialize here. Interpretation goes in a `notes/` page. + +## Source + +- URL: https://x.com/NOTimothyLottes/status/2064858927829745887 +- Author: NOTimothyLottes (@NOTimothyLottes) +- Posted: 2026-06-10 23:55:06 + +## Thread + +**1/** + +So I'm on Linux now, but obviously I cheated, because this is a SteamOS machine (pre-installed). Need to re-learn enough to be able to make responsible choices on a new install. First up what init system to do? {openrc, runit, s6, or dinit}. + +Branches: [[2026-06-11-RouaniJihad-my-patience-is-this-thin-with-win11-right-now-but]], [[2026-06-11-mcnabbd-fascinating-to-me-how-this-challenge-would-have]], [[2026-06-11-NOTimothyLottes-going-to-give-artix-lxqt-runit-variation-a-try]] + +**2/** + +'dinit' is out (no C++ for me) - so runit wins by default [also future desire to run busybox, so it's the natural choice] diff --git a/archive/threads/NOTimothyLottes/index.md b/archive/threads/NOTimothyLottes/index.md new file mode 100644 index 0000000..c6bc6c2 --- /dev/null +++ b/archive/threads/NOTimothyLottes/index.md @@ -0,0 +1,13 @@ +--- +title: NOTimothyLottes +type: note +status: published +draft: false +description: Archived threads by NOTimothyLottes. +tags: + - archive + - twitter + - NOTimothyLottes +--- + +- [[2026-06-10-steamos-init]] diff --git a/archive/threads/index.md b/archive/threads/index.md index f0e07c1..3122857 100644 --- a/archive/threads/index.md +++ b/archive/threads/index.md @@ -13,3 +13,5 @@ Threads are filed by author handle, then by date and title. The main note in eac How to read a folder: [[How threads are organized]]. Example of the note shape (single spine, no branches): [[How to archive a thread]] + +- [[NOTimothyLottes]] diff --git a/assets/README.md b/assets/README.md new file mode 100644 index 0000000..cdd932d --- /dev/null +++ b/assets/README.md @@ -0,0 +1 @@ +Working media for Threadwell. Mirrors archive/threads/// plus handles//. Gitignored except this file. Not published. Lift to the image host later. Do not link notes at this folder. diff --git a/meta/conventions.md b/meta/conventions.md index 8106343..1c17cab 100644 --- a/meta/conventions.md +++ b/meta/conventions.md @@ -39,6 +39,7 @@ draft: true | `meta/` | no | Vault rules, local attachments | | `assets/` | no, gitignored | Working media before upload | | `secrets/` | no, gitignored | Cookies and image-host credentials | +| `scripts/` | no | Tracked tooling. Quartz-ignored | | `private/` | no, gitignored | Never publish, never push | | `site/` | no, gitignored | Local Quartz clone (own remotes) | | `publish/` | no | Tracked Quartz overlay | diff --git a/publish/quartz.config.yaml b/publish/quartz.config.yaml index 13f8a81..37b1ca7 100644 --- a/publish/quartz.config.yaml +++ b/publish/quartz.config.yaml @@ -14,6 +14,9 @@ configuration: - private - templates - meta + - assets + - secrets + - scripts - .obsidian - publish - README.md diff --git a/scripts/twitter/README.md b/scripts/twitter/README.md new file mode 100644 index 0000000..3553da7 --- /dev/null +++ b/scripts/twitter/README.md @@ -0,0 +1,13 @@ +# twitter + +Emit Threadwell archive notes from a `thread_data.json` dump. No fetch, no lift. + +## One dump + +``` +python C:\projects\Threadwell\scripts\twitter\emit_archive.py --input C:\projects\manual_slop\docs\twitter\2064858927829745887 --vault C:\projects\Threadwell --slug steamos-init --archived 2026-08-23 +``` + +`--input` is the dump directory. `--vault` is the Threadwell root. `--slug` overrides the directory slug. `--archived` is `YYYY-MM-DD` and defaults to today. + +The converter writes the spine and branch notes under `archive/threads///` and a working copy under `assets/threads///` (`thread_data.json`, `media.json`, `gaps.md`, and any local media). Media is not published. diff --git a/scripts/twitter/__init__.py b/scripts/twitter/__init__.py new file mode 100644 index 0000000..cbc350e --- /dev/null +++ b/scripts/twitter/__init__.py @@ -0,0 +1 @@ +"""Emit Threadwell archive notes from a thread_data.json dump. No fetch, no lift.""" diff --git a/scripts/twitter/emit_archive.py b/scripts/twitter/emit_archive.py new file mode 100644 index 0000000..5a864b0 --- /dev/null +++ b/scripts/twitter/emit_archive.py @@ -0,0 +1,249 @@ +"""Emit Threadwell archive notes from a thread_data.json dump. No fetch, no lift.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +from datetime import date +from pathlib import Path +from urllib.parse import parse_qs, unquote, urlparse + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from twitter.models import ThreadData, load_thread +from twitter.render import render_branch, render_spine +from twitter.slug import branch_file_name, date_prefix, thread_dir_name +from twitter.tree import by_id, branch_roots, children_map, descendants, spine_ids + + +def _url_media_id(url: str, index: int) -> str: + segment = Path(unquote(urlparse(url).path)).name + return segment if segment else f"m{index}" + + +def _url_ext(url: str) -> str | None: + parsed = urlparse(url) + fmt = parse_qs(parsed.query).get("format", [None])[0] + if fmt: + return fmt.lstrip(".").lower() + suffix = Path(unquote(parsed.path)).suffix + if suffix: + return suffix.lstrip(".").lower() + return None + + +def _media_files(media_dir: Path) -> list[Path]: + if not media_dir.is_dir(): + return [] + return sorted(p for p in media_dir.iterdir() if p.is_file()) + + +def _take_preferred(unused: list[Path], post_id: str, index: int) -> Path | None: + want = f"{post_id}_{index}" + for path in unused: + if path.stem == want: + return path + return None + + +def _take_fallback(unused: list[Path], post_id: str) -> Path | None: + prefix = f"{post_id}_" + for path in unused: + if path.name.startswith(prefix): + return path + return None + + +def collect_media( + thread: ThreadData, + input_dir: Path, + dest_dir: Path, +) -> tuple[list[dict[str, object]], dict[str, tuple[str, ...]]]: + unused = _media_files(input_dir / "media") + items: list[dict[str, object]] = [] + by_post: dict[str, list[str]] = {} + + for post in thread.posts: + for index, url in enumerate(post.media_urls, start=1): + media_id = _url_media_id(url, index) + src = _take_preferred(unused, post.post_id, index) + if src is None: + src = _take_fallback(unused, post.post_id) + if src is not None: + unused.remove(src) + ext = src.suffix.lstrip(".") or _url_ext(url) or "bin" + else: + ext = _url_ext(url) or "bin" + filename = f"{post.post_id}_{media_id}_orig.{ext}" + if src is not None: + dest_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dest_dir / filename) + items.append({ + "post_id": post.post_id, + "media_id": media_id, + "handle": post.handle, + "role": "orig", + "filename": filename, + "publish": False, + "url": None, + }) + by_post.setdefault(post.post_id, []).append(filename) + + media_by_post = {pid: tuple(names) for pid, names in by_post.items()} + return items, media_by_post + + +def render_gaps(thread: ThreadData, spine: list[str]) -> str: + ids = by_id(thread) + suggested = spine[-1] if spine else "" + quote_ids = [p.post_id for p in thread.posts if p.quote_of_id] + missing = [ + p.post_id + for p in thread.posts + if p.reply_to_id is not None and p.reply_to_id not in ids + ] + empty = [p.post_id for p in thread.posts if not (p.text or "").strip()] + + lines = [ + "# gaps", + "input: root", + f"suggested_tip: {suggested}", + ] + if quote_ids: + lines.append("quote_of:") + lines.extend(quote_ids) + else: + lines.append("quote_of: unset") + lines.append("missing_reply_to:") + lines.extend(missing) + lines.append("empty_text:") + lines.extend(empty) + return "\n".join(lines) + "\n" + + +def emit( + input_dir: Path, + vault: Path, + slug: str | None, + archived: str, +) -> None: + src = input_dir / "thread_data.json" + if not src.is_file(): + raise SystemExit(f"missing {src}") + + thread = load_thread(src) + if not thread.posts: + raise SystemExit(f"no posts in {src}") + + spine = spine_ids(thread) + roots = branch_roots(thread, spine) + ids = by_id(thread) + kids = children_map(thread) + first = ids[spine[0]] + handle = first.handle + first_line = (first.text or "").split("\n", 1)[0] + dir_name = thread_dir_name(date_prefix(first.timestamp), first_line, slug) + + branch_names: dict[str, str] = {} + for rid in roots: + branch = ids[rid] + fname = branch_file_name( + date_prefix(branch.timestamp), + branch.handle, + branch.text, + ) + name = fname[:-3] if fname.endswith(".md") else fname + branch_names[rid] = name + + note_dir = vault / "archive" / "threads" / handle / dir_name + asset_dir = vault / "assets" / "threads" / handle / dir_name + note_dir.mkdir(parents=True, exist_ok=True) + asset_dir.mkdir(parents=True, exist_ok=True) + + items, media_by_post = collect_media(thread, input_dir, asset_dir) + + (note_dir / "index.md").write_text( + render_spine( + thread, + spine, + roots, + branch_names, + archived, + media_by_post, + ), + encoding="utf-8", + newline="\n", + ) + for rid in roots: + (note_dir / f"{branch_names[rid]}.md").write_text( + render_branch( + thread, + rid, + descendants(rid, kids), + handle, + archived, + media_by_post, + ), + encoding="utf-8", + newline="\n", + ) + + shutil.copy2(src, asset_dir / "thread_data.json") + media_doc = { + "root_post_id": thread.root_post_id, + "items": items, + } + (asset_dir / "media.json").write_text( + json.dumps(media_doc, indent=2) + "\n", + encoding="utf-8", + newline="\n", + ) + (asset_dir / "gaps.md").write_text( + render_gaps(thread, spine), + encoding="utf-8", + newline="\n", + ) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Emit Threadwell archive notes from a thread_data.json dump. " + "No fetch, no lift." + ), + ) + parser.add_argument( + "--input", + required=True, + type=Path, + help="Dump directory containing thread_data.json", + ) + parser.add_argument( + "--vault", + required=True, + type=Path, + help="Threadwell vault root", + ) + parser.add_argument( + "--slug", + default=None, + help="Optional slug override for the thread directory", + ) + parser.add_argument( + "--archived", + default=date.today().isoformat(), + help="Archive date YYYY-MM-DD (default: today local)", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + emit(args.input, args.vault, args.slug, args.archived) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/twitter/models.py b/scripts/twitter/models.py new file mode 100644 index 0000000..cafdaea --- /dev/null +++ b/scripts/twitter/models.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +import json + + +@dataclass(frozen=True) +class PostMetrics: + reply_count: int + repost_count: int + like_count: int + view_count: int | None + + +@dataclass(frozen=True) +class PostData: + post_id: str + author: str + handle: str + text: str + timestamp: str + media_urls: tuple[str, ...] + reply_to_id: str | None + quote_of_id: str | None + metrics: PostMetrics + + +@dataclass(frozen=True) +class ThreadData: + root_post_id: str + posts: tuple[PostData, ...] + source_url: str + + +def _norm_handle(raw: str) -> str: + h = (raw or "").strip() + return h[1:] if h.startswith("@") else h + + +def _norm_id(raw: object) -> str | None: + if raw is None: + return None + s = str(raw).strip() + return s or None + + +def load_thread(path: Path) -> ThreadData: + data = json.loads(path.read_text(encoding="utf-8")) + posts: list[PostData] = [] + for pd in data.get("posts", []): + m = pd.get("metrics") or {} + view = m.get("view_count") + posts.append(PostData( + post_id=str(pd.get("post_id") or ""), + author=str(pd.get("author") or ""), + handle=_norm_handle(str(pd.get("handle") or "")), + text=str(pd.get("text") or ""), + timestamp=str(pd.get("timestamp") or ""), + media_urls=tuple(pd.get("media_urls") or []), + reply_to_id=_norm_id(pd.get("reply_to_id")), + quote_of_id=_norm_id(pd.get("quote_of_id")), + metrics=PostMetrics( + reply_count=int(m.get("reply_count") or 0), + repost_count=int(m.get("repost_count") or 0), + like_count=int(m.get("like_count") or 0), + view_count=int(view) if view is not None else None, + ), + )) + return ThreadData( + root_post_id=str(data.get("root_post_id") or ""), + posts=tuple(posts), + source_url=str(data.get("source_url") or ""), + ) diff --git a/scripts/twitter/render.py b/scripts/twitter/render.py new file mode 100644 index 0000000..0cab7f0 --- /dev/null +++ b/scripts/twitter/render.py @@ -0,0 +1,217 @@ +"""Render archive note markdown from thread JSON. Strings only; no file I/O.""" + +from __future__ import annotations + +from .models import PostData, ThreadData +from .slug import date_prefix +from .tree import by_id + +_PROVENANCE = ( + "> [!info] Provenance\n" + "> Primary source record. Do not editorialize here. " + "Interpretation goes in a `notes/` page." +) + + +def _yaml_quote(value: str) -> str: + return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def _first_line(text: str) -> str: + return (text or "").split("\n", 1)[0].rstrip("\r") + + +def _title_text(text: str) -> str: + line = _first_line(text) + idx = line.find(". ") + if idx >= 0: + return line[: idx + 1] + return line + + +def _status_url(handle: str, post_id: str) -> str: + return f"https://x.com/{handle}/status/{post_id}" + + +def _blockquote(text: str) -> str: + lines = (text or "").split("\n") + return "\n".join(">" if line == "" else f"> {line}" for line in lines) + + +def _frontmatter( + *, + title: str, + source_url: str, + author: str, + handle: str, + post_id: str, + date: str, + archived: str, + spine_handle: str, + in_reply_to: str, + parent_post_id: str | None = None, +) -> str: + lines = [ + "---", + f"title: {_yaml_quote(title)}", + "type: archive", + "source: twitter", + f"source_url: {_yaml_quote(source_url)}", + f"author: {_yaml_quote(author)}", + f"handle: {handle}", + f"post_id: {_yaml_quote(post_id)}", + f"date: {date}", + f"archived: {archived}", + "status: draft", + "draft: true", + "tags:", + " - archive", + " - twitter", + f" - {spine_handle}", + f"description: {_yaml_quote(title)}", + f"in_reply_to: {_yaml_quote(in_reply_to)}", + ] + if parent_post_id is not None: + lines.append(f"parent_post_id: {_yaml_quote(parent_post_id)}") + lines.append("---") + return "\n".join(lines) + + +def _source_block(post: PostData) -> str: + url = _status_url(post.handle, post.post_id) + return ( + "## Source\n" + "\n" + f"- URL: {url}\n" + f"- Author: {post.author} (@{post.handle})\n" + f"- Posted: {post.timestamp}" + ) + + +def _post_block( + n: int, + post: PostData, + media_by_post: dict[str, tuple[str, ...]] | None, + branch_links: list[str] | None = None, +) -> str: + parts = [f"**{n}/**", "", post.text] + media = (media_by_post or {}).get(post.post_id) or () + if media: + parts.append("") + parts.append("Media (not lifted): " + " ".join(f"`{fn}`" for fn in media)) + if branch_links: + parts.append("") + parts.append("Branches: " + ", ".join(f"[[{name}]]" for name in branch_links)) + parts.append("") + return "\n".join(parts) + + +def _foreign_parent( + first: PostData, + ids: dict[str, PostData], +) -> tuple[str, PostData | None]: + parent_id = first.reply_to_id or "" + if not parent_id: + return "", None + parent = ids.get(parent_id) + if parent is None: + return parent_id, None + if parent.handle != first.handle: + return parent_id, parent + return "", None + + +def render_spine( + thread: ThreadData, + spine: list[str], + branch_root_ids: list[str], + branch_names: dict[str, str], + archived: str, + media_by_post: dict[str, tuple[str, ...]] | None = None, +) -> str: + """branch_names maps branch-root post_id -> wikilink target (filename without .md).""" + ids = by_id(thread) + first = ids[spine[0]] + spine_handle = first.handle + in_reply_to, parent = _foreign_parent(first, ids) + title = _title_text(first.text) + source_url = _status_url(first.handle, first.post_id) + chunks = [ + _frontmatter( + title=title, + source_url=source_url, + author=first.author, + handle=first.handle, + post_id=first.post_id, + date=date_prefix(first.timestamp), + archived=archived, + spine_handle=spine_handle, + in_reply_to=in_reply_to, + ), + "", + _PROVENANCE, + "", + _source_block(first), + "", + ] + if parent is not None: + chunks.append(_blockquote(parent.text)) + chunks.append("") + chunks.append("## Thread") + chunks.append("") + + branches_under: dict[str, list[str]] = {} + for rid in branch_root_ids: + parent_id = ids[rid].reply_to_id + if parent_id: + branches_under.setdefault(parent_id, []).append(branch_names[rid]) + + for n, pid in enumerate(spine, 1): + post = ids[pid] + chunks.append(_post_block(n, post, media_by_post, branches_under.get(pid))) + + return "\n".join(chunks).rstrip() + "\n" + + +def render_branch( + thread: ThreadData, + branch_root_id: str, + descendant_ids: list[str], + spine_handle: str, + archived: str, + media_by_post: dict[str, tuple[str, ...]] | None = None, +) -> str: + """descendant_ids are posts after the root, in display order (tree.descendants).""" + ids = by_id(thread) + root = ids[branch_root_id] + title = _title_text(root.text) + source_url = _status_url(root.handle, root.post_id) + chunks = [ + _frontmatter( + title=title, + source_url=source_url, + author=root.author, + handle=root.handle, + post_id=root.post_id, + date=date_prefix(root.timestamp), + archived=archived, + spine_handle=spine_handle, + in_reply_to="", + parent_post_id=root.reply_to_id or "", + ), + "", + _PROVENANCE, + "", + _source_block(root), + "", + "## Branch", + "", + ] + posts = [root] + [ids[i] for i in descendant_ids] + for n, post in enumerate(posts, 1): + chunks.append(_post_block(n, post, media_by_post)) + chunks.append("## Related") + chunks.append("") + chunks.append("- Spine: [[index]]") + chunks.append("") + return "\n".join(chunks).rstrip() + "\n" diff --git a/scripts/twitter/slug.py b/scripts/twitter/slug.py new file mode 100644 index 0000000..1e080b2 --- /dev/null +++ b/scripts/twitter/slug.py @@ -0,0 +1,34 @@ +from __future__ import annotations +import re + +_MENTION = re.compile(r"^(@[A-Za-z0-9_]+(\s+|$))+") +_APOS = re.compile(r"['\u2019\u2018]") +_NON_SLUG = re.compile(r"[^a-z0-9]+") + +def date_prefix(timestamp: str) -> str: + if len(timestamp) >= 10 and timestamp[4] == "-" and timestamp[7] == "-": + return timestamp[:10] + return "unknown-date" + +def slugify(text: str, max_len: int = 50) -> str: + s = _MENTION.sub("", (text or "").strip()).lower() + s = _APOS.sub("", s) + s = _NON_SLUG.sub("-", s).strip("-") + if not s: + return "thread" + if len(s) <= max_len: + return s + cut = s[:max_len] + if cut.endswith("-"): + return cut.rstrip("-") + cut = cut.rstrip("-") + if "-" in cut: + cut = cut.rsplit("-", 1)[0] + return cut or s[:max_len] + +def thread_dir_name(date: str, first_spine_line: str, override: str | None) -> str: + slug = override.strip() if override else slugify(first_spine_line) + return f"{date}-{slug}" + +def branch_file_name(date: str, handle: str, first_line: str) -> str: + return f"{date}-{handle}-{slugify(first_line)}.md" diff --git a/scripts/twitter/tree.py b/scripts/twitter/tree.py new file mode 100644 index 0000000..576b056 --- /dev/null +++ b/scripts/twitter/tree.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from .models import ThreadData, PostData + + +def by_id(thread: ThreadData) -> dict[str, PostData]: + return {p.post_id: p for p in thread.posts} + + +def children_map(thread: ThreadData) -> dict[str, list[str]]: + ids = by_id(thread) + kids: dict[str, list[str]] = {p.post_id: [] for p in thread.posts} + for p in thread.posts: + if p.reply_to_id and p.reply_to_id in ids: + kids[p.reply_to_id].append(p.post_id) + for parent, lst in kids.items(): + lst.sort(key=lambda i: (ids[i].timestamp, ids[i].post_id)) + return kids + + +def spine_ids(thread: ThreadData) -> list[str]: + ids = by_id(thread) + kids = children_map(thread) + start = thread.root_post_id if thread.root_post_id in ids else thread.posts[0].post_id + handle = ids[start].handle + out = [start] + cur = start + while True: + same = [c for c in kids[cur] if ids[c].handle == handle] + if not same: + break + cur = same[0] + out.append(cur) + return out + + +def branch_roots(thread: ThreadData, spine: list[str]) -> list[str]: + spine_set = set(spine) + ids = by_id(thread) + roots: list[str] = [] + for p in thread.posts: + if p.post_id in spine_set: + continue + parent_on_spine = p.reply_to_id in spine_set + parent_missing = p.reply_to_id is not None and p.reply_to_id not in ids + if parent_on_spine or parent_missing: + roots.append(p.post_id) + roots.sort(key=lambda i: (ids[i].timestamp, ids[i].post_id)) + return roots + + +def descendants(start_id: str, kids: dict[str, list[str]]) -> list[str]: + out: list[str] = [] + stack = list(reversed(kids.get(start_id, []))) + while stack: + n = stack.pop() + out.append(n) + stack.extend(reversed(kids.get(n, []))) + return out diff --git a/secrets/README.md b/secrets/README.md new file mode 100644 index 0000000..0514f6c --- /dev/null +++ b/secrets/README.md @@ -0,0 +1 @@ +Put cookies.txt (Netscape, for gallery-dl) and catbox.env here. Gitignored except this file. Never commit secrets. Never paste them into chat.