more script stuff
This commit is contained in:
ed committed
1 parent
2bce73ccf3
commit
3d07098bd0
6 files changed
+141
-41
No files matched your search
@@ -0,0 +1,104 @@
|
||||
# Bulk, incremental backup of vault assets/threads/* to the configured remote.
|
||||
#
|
||||
# Reads [backup].root from secrets/credentials.toml. The remote path is never
|
||||
# echoed or logged — handle it like any other secret.
|
||||
#
|
||||
# Behaviour: robocopy /E mirror (add or refresh, never delete). Re-runs only
|
||||
# push deltas. assets/WIP/ is deliberately excluded — it is local-only.
|
||||
#
|
||||
# Robocopy exit codes we accept as success: 0 (no change), 1 (files pushed),
|
||||
# 2 / 3 (extras on destination, kept). 4-7 are transient / partial — re-run
|
||||
# after the warning. 8+ are hard fails.
|
||||
|
||||
[CmdletBinding()]
|
||||
param()
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# Resolve paths relative to this script.
|
||||
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$Vault = (Resolve-Path (Join-Path $ScriptDir '..')).Path
|
||||
$Source = Join-Path $Vault 'assets'
|
||||
$ConfigFile = Join-Path $Vault 'secrets\config.toml'
|
||||
|
||||
if (-not (Test-Path -LiteralPath $Source)) {
|
||||
Write-Error "source not found: $Source"
|
||||
exit 2
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $ConfigFile)) {
|
||||
Write-Error "config file not found: $ConfigFile"
|
||||
exit 2
|
||||
}
|
||||
|
||||
# Parse [backup].root from config.toml. Handles both TOML basic strings
|
||||
# ("...", backslashes escaped) and literal strings ('...', backslashes literal).
|
||||
$lines = Get-Content -LiteralPath $ConfigFile
|
||||
$inBackup = $false
|
||||
$rawValue = $null
|
||||
foreach ($line in $lines) {
|
||||
$trimmed = $line.Trim()
|
||||
if ($trimmed -match '^\[(.+)\]$') {
|
||||
$inBackup = ($Matches[1] -eq 'backup')
|
||||
continue
|
||||
}
|
||||
if ($inBackup -and $trimmed -match '^root\s*=\s*(.*)$') {
|
||||
$rawValue = $Matches[1].Trim()
|
||||
break
|
||||
}
|
||||
}
|
||||
if (-not $rawValue) {
|
||||
Write-Error "[backup].root missing in $CredFile"
|
||||
exit 2
|
||||
}
|
||||
|
||||
$remote = $null
|
||||
if ($rawValue.StartsWith("'") -and $rawValue.EndsWith("'") -and $rawValue.Length -ge 2) {
|
||||
$remote = $rawValue.Substring(1, $rawValue.Length - 2)
|
||||
} elseif ($rawValue.StartsWith('"') -and $rawValue.EndsWith('"') -and $rawValue.Length -ge 2) {
|
||||
$inner = $rawValue.Substring(1, $rawValue.Length - 2)
|
||||
try {
|
||||
$remote = [System.Text.RegExp.Regex]::Unescape($inner)
|
||||
} catch {
|
||||
Write-Error "[backup].root has unparseable escapes"
|
||||
exit 2
|
||||
}
|
||||
} else {
|
||||
Write-Error "[backup].root must be a quoted TOML string"
|
||||
exit 2
|
||||
}
|
||||
if (-not $remote) {
|
||||
Write-Error "[backup].root is empty"
|
||||
exit 2
|
||||
}
|
||||
|
||||
# Reachability check — do not echo the path on failure.
|
||||
if (-not (Test-Path -LiteralPath $remote)) {
|
||||
Write-Error "destination unreachable (check [backup].root in credentials.toml)"
|
||||
exit 3
|
||||
}
|
||||
|
||||
# /E copy subdirs incl. empty
|
||||
# /XD WIP exclude the local-only WIP directory
|
||||
# /R:3 /W:5 retries with backoff
|
||||
# /MT:4 multithread (conservative for UNC)
|
||||
# /NP /NDL /NFL keep the log short — no progress, no directory list, no per-file list
|
||||
#
|
||||
# Robocopy's banner prints the destination path. Suppress ALL its output
|
||||
# to keep the secret off stdout/stderr. Exit code is preserved.
|
||||
robocopy $Source $remote /E /XD WIP /R:3 /W:5 /MT:4 /NP /NDL /NFL *>&1 | Out-Null
|
||||
$rc = $LASTEXITCODE
|
||||
|
||||
switch ($rc) {
|
||||
0 { Write-Host "backup ok (no change)" }
|
||||
1 { Write-Host "backup ok (files pushed)" }
|
||||
2 { Write-Host "backup ok (extras on destination, kept)" }
|
||||
3 { Write-Host "backup ok (files pushed, extras on destination, kept)" }
|
||||
{ 4 -le $_ -and $_ -le 7 } {
|
||||
Write-Warning "backup partial (code $rc) — re-run to retry"
|
||||
exit 0
|
||||
}
|
||||
default {
|
||||
Write-Error "backup failed (code $rc)"
|
||||
exit $rc
|
||||
}
|
||||
}
|
||||
+29
-25
@@ -1,38 +1,42 @@
|
||||
# twitter
|
||||
|
||||
Front door: `tw.py`. Run from the vault root.
|
||||
`tw.py`. Run from the vault root.
|
||||
|
||||
```
|
||||
python scripts/twitter/tw.py graph --id <snowflake>
|
||||
python scripts/twitter/tw.py refresh --id <spine-tip> --tip
|
||||
python scripts/twitter/tw.py refresh --id <spine-tip> --tip --preserve-existing
|
||||
python scripts/twitter/tw.py refresh --id <spine-tip> --tip --no-quotes
|
||||
python scripts/twitter/tw.py refresh --id <spine-tip> --tip --branch <branch-tip>
|
||||
python scripts/twitter/tw.py add-branch --id <original-spine-tip> --from <reply-node>
|
||||
python scripts/twitter/tw.py emit --id <spine-tip> --tip --preserve-existing
|
||||
python scripts/twitter/tw.py audit-media --id <snowflake>
|
||||
python scripts/twitter/tw.py publish --id <snowflake>
|
||||
python scripts/twitter/tw.py sync --handle <handle>
|
||||
python scripts/twitter/tw.py backup --id <snowflake>
|
||||
python scripts/twitter/tw.py graph --id <snowflake>
|
||||
python scripts/twitter/tw.py refresh --id <spine-tip> --tip
|
||||
python scripts/twitter/tw.py refresh --id <spine-tip> --tip --preserve-existing
|
||||
python scripts/twitter/tw.py refresh --id <spine-tip> --tip --no-quotes
|
||||
python scripts/twitter/tw.py refresh --id <spine-tip> --tip --branch <branch-tip>
|
||||
python scripts/twitter/tw.py add -branch --id <original-spine-tip> --from <reply-node>
|
||||
python scripts/twitter/tw.py emit --id <spine-tip> --tip --preserve-existing
|
||||
python scripts/twitter/tw.py audit -media --id <snowflake>
|
||||
python scripts/twitter/tw.py publish --id <snowflake>
|
||||
python scripts/twitter/tw.py sync --handle <handle>
|
||||
python scripts/twitter/tw.py backup --id <snowflake>
|
||||
python scripts/twitter/tw.py fallback --id <snowflake> --media-id <id> --role <role> --confirm-origin-unavailable
|
||||
python scripts/twitter/tw.py restore-origin --id <snowflake> --media-id <id>
|
||||
python scripts/twitter/tw.py migrate-media --id <snowflake>
|
||||
python scripts/twitter/tw.py reslug --all
|
||||
python scripts/twitter/tw.py reslug --all --apply
|
||||
python scripts/twitter/tw.py relabel --all
|
||||
python scripts/twitter/tw.py relabel --all --apply
|
||||
python scripts/twitter/tw.py restore -origin --id <snowflake> --media-id <id>
|
||||
python scripts/twitter/tw.py migrate -media --id <snowflake>
|
||||
python scripts/twitter/tw.py reslug --all
|
||||
python scripts/twitter/tw.py reslug --all --apply
|
||||
python scripts/twitter/tw.py relabel --all
|
||||
python scripts/twitter/tw.py relabel --all --apply
|
||||
```
|
||||
|
||||
`paths.py` finds the vault two parents above this file. Dumps live at `../manual_slop/docs/twitter`. Scratch lives at `../Threadwell-ai/scratch`.
|
||||
|
||||
## Capture
|
||||
|
||||
`refresh` is refetch + ingest + emit `--force` + media merge. `--tip` treats `--id` as the tip and walks back to the opening post. If that tip is the OP, same-handle self-replies stay on the spine and foreign replies are branches. Repeat `--branch <tip-id>` to capture extra tips into the same `thread_data.json`. `--attach <child>:<parent>` sets one hop before the walk. A walk that stops on a missing parent aborts unless `--allow-broken-walk`. A leftover dir under another handle aborts unless `--retire-old-dir`. It does not invent replies. Captures run one at a time. gallery-dl retries are off. Extractor and request sleeps are five seconds. Re-emit deletes branch notes that are no longer roots.
|
||||
`refresh` is refetch + ingest + emit `--force` + media merge. `--tip` treats `--id` as the tip and walks back to the opening post.
|
||||
If that tip is the OP, same-handle self-replies stay on the spine and foreign replies are branches.
|
||||
Repeat `--branch <tip-id>` to capture extra tips into the same `thread_data.json`. `--attach <child>:<parent>` sets one hop before the walk.
|
||||
A walk that stops on a missing parent aborts unless `--allow-broken-walk`.
|
||||
A leftover dir under another handle aborts unless `--retire-old-dir`.
|
||||
It does not invent replies. Captures run one at a time. gallery-dl retries are off.
|
||||
Extractor and request sleeps are five seconds. Re-emit deletes branch notes that are no longer roots.
|
||||
|
||||
`refresh` also captures each quote tweet on the quoter spine as
|
||||
its own root archive (`refresh --id <quoted>` without `--tip`).
|
||||
One hop. `--no-quotes` skips that pass. `emit` does not fetch.
|
||||
Established threads: `refresh --id <quoter-tip> --tip --preserve-existing`.
|
||||
`refresh` also captures each quote tweet on the quoter spine as its own root archive (`refresh --id <quoted>` without `--tip`).
|
||||
One hop. `--no-quotes` skips that pass. `emit` does not fetch. Established threads: `refresh --id <quoter-tip> --tip --preserve-existing`.
|
||||
|
||||
Each thread writes one archive folder. The owner is the opening post (`reply_to_id` is empty). Other authors stay in that folder. New notes set `draft: false`. The command does not commit. Frozen ids abort.
|
||||
|
||||
@@ -68,11 +72,11 @@ A thread folder is named from the title in `index.md`: `<YYYY-MM-DD>-<slug>`. `r
|
||||
|
||||
Later emits use that same title for the folder and the frontmatter. Move both sides together. Each old archive prefix is replaced once in mutable `*.md` and `*.canvas` outside `.git/`, `site/`, `assets/`, `secrets/`, `node_modules/`, and frozen archive dirs.
|
||||
|
||||
`relabel --all` patches existing `**N/**` lines to `**N/** @handle` from on-disk `thread_data.json`. No scrape. No folder rename. `--apply` writes notes only.
|
||||
`relabel --all` patches existing `**N/**` lines to `**N/** @handle` from on-disk `thread_data.json`. `--apply` writes notes only.
|
||||
|
||||
## Secrets
|
||||
|
||||
`secrets/twitter_cookies.txt` and `secrets/credentials.toml` stay on the machine that runs the capture. The scripts read them. They never print them. The Catbox hash goes in an HTTPS form body, not on a process command line. The backup command prints `synced` or `error`, not the destination path.
|
||||
`secrets/twitter_cookies.txt` and `secrets/credentials.toml` stay on the machine that runs the capture. Catbox hash goes in an HTTPS form body. Backup command prints `synced` or `error`.
|
||||
|
||||
## Pieces
|
||||
|
||||
|
||||
@@ -42,8 +42,8 @@ class BackupResult:
|
||||
|
||||
|
||||
def load_destination_root(vault: Path, destination_id: str) -> Path:
|
||||
"""Read the configured backup destination root from ``vault/secrets/credentials.toml``."""
|
||||
path = vault / "secrets" / "credentials.toml"
|
||||
"""Read the configured backup destination root from ``vault/secrets/config.toml``."""
|
||||
path = vault / "secrets" / "config.toml"
|
||||
if not path.is_file():
|
||||
raise RuntimeError(f"missing {path}")
|
||||
text = path.read_text(encoding="utf-8")
|
||||
@@ -68,7 +68,7 @@ def load_destination_root(vault: Path, destination_id: str) -> Path:
|
||||
if configured_id != destination_id:
|
||||
raise RuntimeError(f"backup destination not configured: {destination_id}")
|
||||
if not root:
|
||||
raise RuntimeError("credentials.toml [backup] root is empty")
|
||||
raise RuntimeError("config.toml [backup] root is empty")
|
||||
return Path(root)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user