From 6d3a2dd7d9fb9b6b6e91b292614783696e6229e0 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Thu, 19 Feb 2026 16:22:49 -0500 Subject: [PATCH] curation & gather 2 --- references/ASM_NOTES.csv | 70 +++++++++++++++++++++++++++++++ references/grok_search_query_3.md | 40 ++++++++++++++++++ references/grok_search_query_4.md | 64 ++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 references/ASM_NOTES.csv create mode 100644 references/grok_search_query_3.md create mode 100644 references/grok_search_query_4.md diff --git a/references/ASM_NOTES.csv b/references/ASM_NOTES.csv new file mode 100644 index 0000000..0e4f40c --- /dev/null +++ b/references/ASM_NOTES.csv @@ -0,0 +1,70 @@ +"Anchor Text","URL" +"Demystifying programs that create programs","https://briancallahan.net/blog/20210407.html" +"Part 1: A disassembler","https://briancallahan.net/blog/20210407.html" +"Part 2: Starting an assembler","https://briancallahan.net/blog/20210408.html" +"Part 3: Globals, passes, and error handling","https://briancallahan.net/blog/20210409.html" +"Part 4: Parsing","https://briancallahan.net/blog/20210410.html" +"Part 5: Processing our first opcode","https://briancallahan.net/blog/20210411.html" +"Part 6: Processing more opcodes","https://briancallahan.net/blog/20210412.html" +"Part 7: Further opcode processing","https://briancallahan.net/blog/20210413.html" +"Part 8: Finishing opcode processing","https://briancallahan.net/blog/20210414.html" +"Source","https://github.com/ibara/d80" +"Assembly Programming and Computer Architecture for Software Engineers","https://brianrhall.com/book/" +"Performance-Aware Programming Series","https://www.computerenhance.com/p/table-of-contents" +"Instruction Decoding on the 8086","https://www.computerenhance.com/p/instruction-decoding-on-the-8086" +"Decoding Multiple Instructions and Suffixes","https://www.computerenhance.com/p/decoding-multiple-instructions-and" +"Opcode Patterns in 8086 Arithmetic","https://www.computerenhance.com/p/opcode-patterns-in-8086-arithmetic" +"8086 Decoder Code Review","https://www.computerenhance.com/p/8086-decoder-code-review" +"Using the Reference Decoder as a Shared Library","https://www.computerenhance.com/p/using-the-reference-decoder-as-a" +"Simulating Non-memory MOVs","https://www.computerenhance.com/p/simulating-non-memory-movs" +"Simulating ADD, SUB, and CMP","https://www.computerenhance.com/p/simulating-add-jmp-and-cmp" +"Simulating Conditional Jumps","" +"Simulating Memory","" +"Simulating Real Programs","" +"Other Common Instructions","" +"The Stack","" +"From 8086 to x64","" +"8086 Simulation Code Review","" +"Assemblers and Loaders (1993)","https://www.davidsalomon.name/assem.advertis/AssemAd.html" +"Affine Notes","https://app.affine.pro/workspace/3fc9d28f-0ee6-4e4c-9675-79230706f341/koifv--tIzioAWQsC3OY0?mode=edgeless" +"An Introduction to Assembly Programming with RISC-V","https://riscv-programming.org/book/riscv-book.html" +"asm-lessons","https://github.com/FFmpeg/asm-lessons" +"20 part Linker Essay (on LWN)","https://lwn.net/Articles/276782/" +"Linkers and Loaders (2000)","https://linker.iecc.com" +"Beginning x64 Assembly Programming: From Novice to AVX Professional","https://link.springer.com/book/10.1007/978-1-4842-5076-1" +"Zen of Assembly Language","https://www.jagregory.com/abrash-zen-of-asm/" +"Let's Learn x86-64 Assembly!","https://gpfault.net/posts/asm-tut-0.txt.html" +"Part 0 - Setup and First Steps","https://gpfault.net/posts/asm-tut-0.txt.html" +"Part 1 - Metaprogramming in Flat Assembler","https://gpfault.net/posts/asm-tut-1.txt.html" +"Part 2 - We're Writing a Virtual Machine","https://gpfault.net/posts/asm-tut-2.txt.html" +"Part 3 - Arithmetic and Logic","https://gpfault.net/posts/asm-tut-3.txt.html" +"Demystifying Debuggers, Part 2: The Anatomy Of A Running Program","https://www.rfleury.com/p/demystifying-debuggers-part-2-the" +"Zydis","https://github.com/zyantific/zydis" +"ZASM (Zydis frontend, not to be confused with Z80 assembler...)","https://github.com/zyantific/zasm" +"Netwide Assembler","https://www.nasm.us" +"EuroAssembler","https://euroassembler.eu/eadoc/" +"DynASM","https://github.com/Esvandiary/DynASM" +"AssemblyLine","https://github.com/0xADE1A1DE/AssemblyLine" +"AsmJit","https://github.com/asmjit" +"AsmTK","https://github.com/asmjit/asmtk" +"VASM","http://sun.hasenbraten.de/vasm/" +"FASM","https://flatassembler.net" +"YASM (Rewrite (on some) of Netwide in C)","https://yasm.tortall.net" +"Design","https://www.tortall.net/projects/yasm/reference/design/design.pdf" +"User Manual","https://www.tortall.net/projects/yasm/manual/manual.pdf" +"TASM","https://github.com/qb40/tasm" +"HLA","https://plantation-productions.com/Webster/HighLevelAsm/index.html" +"libasm","https://github.com/tgtakaoka/libasm" +"xbyak","https://github.com/herumi/xbyak" +"Keystone assembler framework","https://github.com/keystone-engine/keystone" +"MASM","https://github.com/qb40/masm" +"GAS","https://wiki.osdev.org/GAS" +"LIEF","https://github.com/lief-project/LIEF" +"PE Format","https://learn.microsoft.com/en-us/windows/win32/debug/pe-format?redirectedfrom=MSDN" +"ELF","https://wiki.osdev.org/ELF" +"Mach-O","https://github.com/aidansteele/osx-abi-macho-file-format-reference/blob/master/Mach-O_File_Format.pdf" +"SBCL: the ultimate assembly code breadboard","https://pvk.ca/Blog/2014/03/15/sbcl-the-ultimate-assembly-code-breadboard/" +"mschwartz's assembly tutorial","https://github.com/mschwartz/assembly-tutorial" +"CppCon 2018: Matt Godbolt “The Bits Between the Bits: How We Get to main()”","https://youtu.be/dOfucXtyEsU" +"Learning Assembly YT Playlist","https://www.youtube.com/playlist?list=PLHJns8WZXCdvESvdr1BRjo4RHiR1Ylhw9" +"OSdev Entry","https://wiki.osdev.org/Assembly" \ No newline at end of file diff --git a/references/grok_search_query_3.md b/references/grok_search_query_3.md new file mode 100644 index 0000000..2502d7a --- /dev/null +++ b/references/grok_search_query_3.md @@ -0,0 +1,40 @@ +Here’s **more** highly relevant additions (scavenged fresh, strictly aligned to your zero-overhead sourceless x86-64 ColorForth-derivative: 32-bit token arrays, hex/sourceless editor, tape-drive scatter, 2-reg stack, instant compile/live reload). Strong focus on **WebAssembly** (binary format, linear memory tape parallels, dynamic emission, structured binary editors) plus other immediately useful tooling. No direct new Lottes/Onat Wasm material, but Wasm/SPIR-V binary parallels are explicit. + +### WebAssembly Core Parallels & Tooling +Wasm’s binary section format + single growable linear memory (fixed-offset load/store, no runtime stack) is an almost 1:1 match for your 32-bit aligned tokens + preemptive scatter “tape drive” args. Decode/JIT is designed for <5 ms instantiation. + +- **WAForth** (dynamic Forth-to-Wasm compiler written entirely in raw .wat) + https://github.com/remko/waforth + https://mko.re/waforth/ (live demo console) + ~14 KB (7 KB gz) complete minimal Forth interpreter + **dynamic compiler** that emits new Wasm bytecode on-the-fly (LEB128 append, function table + indirect calls for words). Uses linear memory for 4-byte aligned stack/tape. CODE word for raw Wasm opcodes. Jonesforth-inspired. FOSDEM 2023 talk slides (hand-written minimal Wasm Forth system): https://archive.fosdem.org/2023/schedule/event/webassemblyforth/attachments/slides/5876/export/events/attachments/webassemblyforth/slides/5876/Exploring_WebAssembly_With_Forth.pdf + Direct blueprint for your token → machine-code emission loop and live reload. + +- **ImHex** (star for your sourceless hex-editor frontend) + https://github.com/WerWolv/ImHex + https://web.imhex.werwolv.net/ (browser version runs via Wasm) + Professional hex editor with **Pattern Language** (.hexpat – C-like DSL). Define custom 32-bit token structs (value + tag + annotation overlay) → automatic parse, highlight, tree view, colors, names, disassembly. Live patching, huge files, zero string parsing. Patterns repo: https://github.com/WerWolv/ImHex-Patterns (hundreds of examples). + Fork this for exact Lottes-style annotation overlay on your token array. Native + web versions both zero-dep. + + Tiny example pattern snippet you can drop in (for a 32-bit token + 4-bit tag + 64-bit annotation block): + ```c + struct Token { + u32 value; + u8 tag : 4; + u8 padding : 4; + char name[7]; // 7×7-bit compressed like Lottes + u64 annotation; // editor overlay (color/format) + }; + Token tokens[$ / sizeof(Token)]; + ``` + +- **WABT / wasm-tools** (low-level binary manipulation) + https://github.com/WebAssembly/wabt + Official toolkit: wat2wasm / wasm2wat / disassembly / validation. Use for fast token-dictionary ↔ binary round-tripping or hex-patching prototypes. Pairs perfectly with ImHex. + +- **Visual Wasm Structure Editor** (sourceless module building) + https://www.youtube.com/watch?v=liJg6rzXnPg + Visual (non-text) editor for adding instructions/sections directly to Wasm binary. Exact parallel to hex-editor + annotation for token arrays. + +### Other Immediately Helpful +- **Wasm linear memory as tape-drive reference** + Wasm’s flat byte array with fixed-offset access is the cleanest public example of “preemptive scatter” without a runtime data stack. See practical buffer/tape examples: https://blog.jeremylikness.com/blog/2019-04-22_play-the-chaos-game-to-understand-webassembly-memory-management/ (passing pre-placed arrays between host and Wasm with zero copying). diff --git a/references/grok_search_query_4.md b/references/grok_search_query_4.md new file mode 100644 index 0000000..f0493fc --- /dev/null +++ b/references/grok_search_query_4.md @@ -0,0 +1,64 @@ +**Yes – minimal but noted.** + +https://github.com/guitarvydas/forthish is a **low-activity educational repo** (26 commits, 1 star, last touched years ago) containing **progressive Python prototypes** that teach Forth internals by building up from scratch. + +### What aligns (conceptually useful for your dictionary / threading baseline before x86 asm) +- **Direct Threaded Code (DTC)** implementation (explicitly called out in `fcomp.py`): + > "If you care, this is effectively a DTC (Direct Threaded Code) implementation." + +- **Execution tokens (`xt`)** + classic dictionary layout (name field / link field / code field) in `fram.py`. + +- **Compilation state machine** (`state` var: 0=interpret, 1=compile) + colon definitions (`:`, `;`, `if`/`else`/`then`, literals, branches) in `fcomp.py`. + +- Progressive layering: + `simple.py` (bare stack + parser) + `fram.py` (dictionary + `xt` + `execute`) + `fvars.py` (memory, `!` `@` `,`) + `fcomp.py` (full user words + DTC) + +No ColorForth, no 32-bit tokens/tags, no sourceless, no hex-editor, no tape-drive scatter, no 2-reg stack, no x86-64/machine-code, no live reload, no Lottes/Onat mentions. + +It does link out to things you already have (colorforth.github.io, JonesForth, Chuck Moore papers). + +**Bottom line:** Pure high-level teaching aid for understanding DTC / `xt` / dictionary mechanics in a readable Python form. Not a direct blueprint for your zero-overhead x86 token array + hex frontend, but could serve as a clean reference if you want to sketch the threading model before dropping it into asm. + +That’s everything worthwhile in the repo. Nothing deeper. + +--- + +**Yes.** + +Zydis **fully supports encoding** (code generation / assembling raw bytes from instruction structs) via the stable `ZydisEncoder` API. + +- Core library (not separate module) +- Production-ready since ~v4.0 (current stable: v4.1.1 as of Feb 2025) +- Zero dynamic allocation, thread-safe, tiny overhead — perfect fit for your instant <5 ms token → x86-64 emission in the sourceless ColorForth derivative +- Companion **zasm** (https://github.com/zyantific/zasm) gives AsmJit-style C++ frontend on top of the same encoder (mentioned in your ASM_Notes) + +### Minimal encoding example (C, direct from official README) +```c +ZydisEncoderRequest req; +memset(&req, 0, sizeof(req)); + +req.mnemonic = ZYDIS_MNEMONIC_MOV; +req.machine_mode = ZYDIS_MACHINE_MODE_LONG_64; +req.operand_count = 2; +req.operands[0].type = ZYDIS_OPERAND_TYPE_REGISTER; +req.operands[0].reg.value = ZYDIS_REGISTER_RAX; +req.operands[1].type = ZYDIS_OPERAND_TYPE_IMMEDIATE; +req.operands[1].imm.u = 0x1337; + +ZyanU8 encoded[ZYDIS_MAX_INSTRUCTION_LENGTH]; +ZyanUSize len = sizeof(encoded); + +if (ZYAN_SUCCESS(ZydisEncoderEncodeInstruction(&req, encoded, &len))) { + // encoded now holds e.g. 48 C7 C0 37 13 00 00 +} +``` + +This is exactly the kind of zero-overhead emitter you can drop into your 32-bit token compiler loop (pair with Zydis decoder for round-tripping if needed). + +Full docs: https://doc.zydis.re/v4.1.0/html/group__encoder +Repo: https://github.com/zyantific/zydis (examples/ folder has more) + +Nothing else needed unless you want a stripped minimal build snippet or zasm wrapper example. \ No newline at end of file