From 3d07098bd00481df98aaaecbe4b8192d478e1e69 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 26 Aug 2026 22:16:28 -0400 Subject: [PATCH] more script stuff --- README.md | 12 ++-- assets/README.md | 2 +- scripts/mirror_assets.ps1 | 104 +++++++++++++++++++++++++++++++ scripts/twitter/README.md | 54 ++++++++-------- scripts/twitter/backup_assets.py | 6 +- secrets/README.md | 4 -- 6 files changed, 141 insertions(+), 41 deletions(-) create mode 100644 scripts/mirror_assets.ps1 diff --git a/README.md b/README.md index 35aabd7..7720b93 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ Recovered threads and submerged records. An [Obsidian](https://obsidian.md) vault published with [Quartz 5](https://quartz.jzhao.xyz) to GitHub Pages. The site keeps Twitter/X threads and other material that search does not keep. +The goal is archival purposes, for mainly old posts that are hard to scavange, track, or lack of trust in x.com to remain publishing the original content or the user deciding to not preserve said content on-site. Live: [threadwell.edsabode.dev](https://threadwell.edsabode.dev) @@ -28,7 +29,7 @@ notes/ # public — authored writing templates/ # not published meta/ # not published — conventions assets/ # not published, gitignored — working media -secrets/ # not published, gitignored — cookies, host keys +secrets/ # not published, gitignored scripts/ # not published — local tooling (tracked) private/ # not published, not pushed publish/ # tracked overlay: quartz.config.yaml @@ -37,7 +38,7 @@ site/ # local Quartz clone (gitignored; own remotes) .github/workflows/deploy.yml ``` -`archive/`, `canvases/`, and `notes/` publish by default. `site/` is a separate checkout of Quartz (`origin` → `jackyzha0/quartz`). This repo's remote is `Ed94/Threadwell`. +`archive/`, `canvases/`, and `notes/` publish by default. `site/` is a separate checkout of Quartz (`origin` → `jackyzha0/quartz`). ## Frontmatter @@ -69,11 +70,10 @@ Notes cite the original host URL. Keep local copies in `assets/` (not committed). Notes do not point at `assets/` or `./media/`. -YouTube and tweets (Quartz Obsidian-flavored Markdown): +YouTube (Quartz Obsidian-flavored Markdown): ```md ![](https://www.youtube.com/watch?v=VIDEO_ID) -![](https://x.com/user/status/ID) ``` Odysee (iframe; copy the embed URL from Odysee share): @@ -139,7 +139,3 @@ Quartz remotes live in `site/` (`cd site; git pull`). After a Quartz pull, re-co Current major is **Quartz 5**. Edit `publish/quartz.config.yaml`, not the clone. Canvas uses `@quartz-community/canvas-page`. Drafts use `@quartz-community/remove-draft`. Engine license: MIT (Quartz). Vault content: Unlicense (`LICENSE`). - -## Notice - -I'm not a web dev. Look away at the scripts. This exists because it didn't, so it was done badly. diff --git a/assets/README.md b/assets/README.md index ba9851b..3f8d932 100644 --- a/assets/README.md +++ b/assets/README.md @@ -1,5 +1,5 @@ Working media. Same tree as `archive/threads///`, plus `handles//` for avatars. -Gitignored except this file. Not on the site. Notes never link here. +Gitignored except this file. Published images use the original host URL. `tw.py backup` copies this tree to the configured store. `tw.py fallback` uploads one file after the origin is gone. diff --git a/scripts/mirror_assets.ps1 b/scripts/mirror_assets.ps1 new file mode 100644 index 0000000..a0ed04c --- /dev/null +++ b/scripts/mirror_assets.ps1 @@ -0,0 +1,104 @@ +# Bulk, incremental backup of vault assets/threads/* to the configured remote. +# +# Reads [backup].root from secrets/credentials.toml. The remote path is never +# echoed or logged — handle it like any other secret. +# +# Behaviour: robocopy /E mirror (add or refresh, never delete). Re-runs only +# push deltas. assets/WIP/ is deliberately excluded — it is local-only. +# +# Robocopy exit codes we accept as success: 0 (no change), 1 (files pushed), +# 2 / 3 (extras on destination, kept). 4-7 are transient / partial — re-run +# after the warning. 8+ are hard fails. + +[CmdletBinding()] +param() + +$ErrorActionPreference = 'Stop' + +# Resolve paths relative to this script. +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$Vault = (Resolve-Path (Join-Path $ScriptDir '..')).Path +$Source = Join-Path $Vault 'assets' +$ConfigFile = Join-Path $Vault 'secrets\config.toml' + +if (-not (Test-Path -LiteralPath $Source)) { + Write-Error "source not found: $Source" + exit 2 +} +if (-not (Test-Path -LiteralPath $ConfigFile)) { + Write-Error "config file not found: $ConfigFile" + exit 2 +} + +# Parse [backup].root from config.toml. Handles both TOML basic strings +# ("...", backslashes escaped) and literal strings ('...', backslashes literal). +$lines = Get-Content -LiteralPath $ConfigFile +$inBackup = $false +$rawValue = $null +foreach ($line in $lines) { + $trimmed = $line.Trim() + if ($trimmed -match '^\[(.+)\]$') { + $inBackup = ($Matches[1] -eq 'backup') + continue + } + if ($inBackup -and $trimmed -match '^root\s*=\s*(.*)$') { + $rawValue = $Matches[1].Trim() + break + } +} +if (-not $rawValue) { + Write-Error "[backup].root missing in $CredFile" + exit 2 +} + +$remote = $null +if ($rawValue.StartsWith("'") -and $rawValue.EndsWith("'") -and $rawValue.Length -ge 2) { + $remote = $rawValue.Substring(1, $rawValue.Length - 2) +} elseif ($rawValue.StartsWith('"') -and $rawValue.EndsWith('"') -and $rawValue.Length -ge 2) { + $inner = $rawValue.Substring(1, $rawValue.Length - 2) + try { + $remote = [System.Text.RegExp.Regex]::Unescape($inner) + } catch { + Write-Error "[backup].root has unparseable escapes" + exit 2 + } +} else { + Write-Error "[backup].root must be a quoted TOML string" + exit 2 +} +if (-not $remote) { + Write-Error "[backup].root is empty" + exit 2 +} + +# Reachability check — do not echo the path on failure. +if (-not (Test-Path -LiteralPath $remote)) { + Write-Error "destination unreachable (check [backup].root in credentials.toml)" + exit 3 +} + +# /E copy subdirs incl. empty +# /XD WIP exclude the local-only WIP directory +# /R:3 /W:5 retries with backoff +# /MT:4 multithread (conservative for UNC) +# /NP /NDL /NFL keep the log short — no progress, no directory list, no per-file list +# +# Robocopy's banner prints the destination path. Suppress ALL its output +# to keep the secret off stdout/stderr. Exit code is preserved. +robocopy $Source $remote /E /XD WIP /R:3 /W:5 /MT:4 /NP /NDL /NFL *>&1 | Out-Null +$rc = $LASTEXITCODE + +switch ($rc) { + 0 { Write-Host "backup ok (no change)" } + 1 { Write-Host "backup ok (files pushed)" } + 2 { Write-Host "backup ok (extras on destination, kept)" } + 3 { Write-Host "backup ok (files pushed, extras on destination, kept)" } + { 4 -le $_ -and $_ -le 7 } { + Write-Warning "backup partial (code $rc) — re-run to retry" + exit 0 + } + default { + Write-Error "backup failed (code $rc)" + exit $rc + } +} diff --git a/scripts/twitter/README.md b/scripts/twitter/README.md index 385f8b3..0b34341 100644 --- a/scripts/twitter/README.md +++ b/scripts/twitter/README.md @@ -1,38 +1,42 @@ # twitter -Front door: `tw.py`. Run from the vault root. +`tw.py`. Run from the vault root. ``` -python scripts/twitter/tw.py graph --id -python scripts/twitter/tw.py refresh --id --tip -python scripts/twitter/tw.py refresh --id --tip --preserve-existing -python scripts/twitter/tw.py refresh --id --tip --no-quotes -python scripts/twitter/tw.py refresh --id --tip --branch -python scripts/twitter/tw.py add-branch --id --from -python scripts/twitter/tw.py emit --id --tip --preserve-existing -python scripts/twitter/tw.py audit-media --id -python scripts/twitter/tw.py publish --id -python scripts/twitter/tw.py sync --handle -python scripts/twitter/tw.py backup --id +python scripts/twitter/tw.py graph --id +python scripts/twitter/tw.py refresh --id --tip +python scripts/twitter/tw.py refresh --id --tip --preserve-existing +python scripts/twitter/tw.py refresh --id --tip --no-quotes +python scripts/twitter/tw.py refresh --id --tip --branch +python scripts/twitter/tw.py add -branch --id --from +python scripts/twitter/tw.py emit --id --tip --preserve-existing +python scripts/twitter/tw.py audit -media --id +python scripts/twitter/tw.py publish --id +python scripts/twitter/tw.py sync --handle +python scripts/twitter/tw.py backup --id python scripts/twitter/tw.py fallback --id --media-id --role --confirm-origin-unavailable -python scripts/twitter/tw.py restore-origin --id --media-id -python scripts/twitter/tw.py migrate-media --id -python scripts/twitter/tw.py reslug --all -python scripts/twitter/tw.py reslug --all --apply -python scripts/twitter/tw.py relabel --all -python scripts/twitter/tw.py relabel --all --apply +python scripts/twitter/tw.py restore -origin --id --media-id +python scripts/twitter/tw.py migrate -media --id +python scripts/twitter/tw.py reslug --all +python scripts/twitter/tw.py reslug --all --apply +python scripts/twitter/tw.py relabel --all +python scripts/twitter/tw.py relabel --all --apply ``` `paths.py` finds the vault two parents above this file. Dumps live at `../manual_slop/docs/twitter`. Scratch lives at `../Threadwell-ai/scratch`. ## Capture -`refresh` is refetch + ingest + emit `--force` + media merge. `--tip` treats `--id` as the tip and walks back to the opening post. If that tip is the OP, same-handle self-replies stay on the spine and foreign replies are branches. Repeat `--branch ` to capture extra tips into the same `thread_data.json`. `--attach :` sets one hop before the walk. A walk that stops on a missing parent aborts unless `--allow-broken-walk`. A leftover dir under another handle aborts unless `--retire-old-dir`. It does not invent replies. Captures run one at a time. gallery-dl retries are off. Extractor and request sleeps are five seconds. Re-emit deletes branch notes that are no longer roots. +`refresh` is refetch + ingest + emit `--force` + media merge. `--tip` treats `--id` as the tip and walks back to the opening post. +If that tip is the OP, same-handle self-replies stay on the spine and foreign replies are branches. +Repeat `--branch ` to capture extra tips into the same `thread_data.json`. `--attach :` sets one hop before the walk. +A walk that stops on a missing parent aborts unless `--allow-broken-walk`. +A leftover dir under another handle aborts unless `--retire-old-dir`. +It does not invent replies. Captures run one at a time. gallery-dl retries are off. +Extractor and request sleeps are five seconds. Re-emit deletes branch notes that are no longer roots. -`refresh` also captures each quote tweet on the quoter spine as -its own root archive (`refresh --id ` without `--tip`). -One hop. `--no-quotes` skips that pass. `emit` does not fetch. -Established threads: `refresh --id --tip --preserve-existing`. +`refresh` also captures each quote tweet on the quoter spine as its own root archive (`refresh --id ` without `--tip`). +One hop. `--no-quotes` skips that pass. `emit` does not fetch. Established threads: `refresh --id --tip --preserve-existing`. Each thread writes one archive folder. The owner is the opening post (`reply_to_id` is empty). Other authors stay in that folder. New notes set `draft: false`. The command does not commit. Frozen ids abort. @@ -68,11 +72,11 @@ A thread folder is named from the title in `index.md`: `-`. `r Later emits use that same title for the folder and the frontmatter. Move both sides together. Each old archive prefix is replaced once in mutable `*.md` and `*.canvas` outside `.git/`, `site/`, `assets/`, `secrets/`, `node_modules/`, and frozen archive dirs. -`relabel --all` patches existing `**N/**` lines to `**N/** @handle` from on-disk `thread_data.json`. No scrape. No folder rename. `--apply` writes notes only. +`relabel --all` patches existing `**N/**` lines to `**N/** @handle` from on-disk `thread_data.json`. `--apply` writes notes only. ## Secrets -`secrets/twitter_cookies.txt` and `secrets/credentials.toml` stay on the machine that runs the capture. The scripts read them. They never print them. The Catbox hash goes in an HTTPS form body, not on a process command line. The backup command prints `synced` or `error`, not the destination path. +`secrets/twitter_cookies.txt` and `secrets/credentials.toml` stay on the machine that runs the capture. Catbox hash goes in an HTTPS form body. Backup command prints `synced` or `error`. ## Pieces diff --git a/scripts/twitter/backup_assets.py b/scripts/twitter/backup_assets.py index d792191..89d04f3 100644 --- a/scripts/twitter/backup_assets.py +++ b/scripts/twitter/backup_assets.py @@ -42,8 +42,8 @@ class BackupResult: def load_destination_root(vault: Path, destination_id: str) -> Path: - """Read the configured backup destination root from ``vault/secrets/credentials.toml``.""" - path = vault / "secrets" / "credentials.toml" + """Read the configured backup destination root from ``vault/secrets/config.toml``.""" + path = vault / "secrets" / "config.toml" if not path.is_file(): raise RuntimeError(f"missing {path}") text = path.read_text(encoding="utf-8") @@ -68,7 +68,7 @@ def load_destination_root(vault: Path, destination_id: str) -> Path: if configured_id != destination_id: raise RuntimeError(f"backup destination not configured: {destination_id}") if not root: - raise RuntimeError("credentials.toml [backup] root is empty") + raise RuntimeError("config.toml [backup] root is empty") return Path(root) diff --git a/secrets/README.md b/secrets/README.md index 640022d..fbe28ed 100644 --- a/secrets/README.md +++ b/secrets/README.md @@ -1,5 +1 @@ Local credentials. Gitignored except this file. - -The scripts read these files. They never print the values. - -Do not commit this folder. Do not paste the contents into chat or issues.