mirror of
https://github.com/Ed94/Odin.git
synced 2026-07-31 20:00:09 +00:00
Merge branch 'master' into pr/1726
This commit is contained in:
Jeroen van Rijn
@@ -0,0 +1,148 @@
|
||||
/*
|
||||
This file was generated, so don't edit this by hand.
|
||||
Transliterated from https://github.com/Ed-von-Schleck/shoco/blob/master/shoco_model.h,
|
||||
which is an English word model.
|
||||
*/
|
||||
|
||||
// package shoco is an implementation of the shoco short string compressor
|
||||
package shoco
|
||||
|
||||
DEFAULT_MODEL :: Shoco_Model {
|
||||
min_char = 39,
|
||||
max_char = 122,
|
||||
characters_by_id = {
|
||||
'e', 'a', 'i', 'o', 't', 'h', 'n', 'r', 's', 'l', 'u', 'c', 'w', 'm', 'd', 'b', 'p', 'f', 'g', 'v', 'y', 'k', '-', 'H', 'M', 'T', '\'', 'B', 'x', 'I', 'W', 'L',
|
||||
},
|
||||
ids_by_character = {
|
||||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 26, -1, -1, -1, -1, -1, 22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 27, -1, -1, -1, -1, -1, 23, 29, -1, -1, 31, 24, -1, -1, -1, -1, -1, -1, 25, -1, -1, 30, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 15, 11, 14, 0, 17, 18, 5, 2, -1, 21, 9, 13, 6, 3, 16, -1, 7, 8, 4, 10, 19, 12, 28, 20, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
},
|
||||
successors_by_bigram = {
|
||||
7, 4, 12, -1, 6, -1, 1, 0, 3, 5, -1, 9, -1, 8, 2, -1, 15, 14, -1, 10, 11, -1, -1, -1, -1, -1, -1, -1, 13, -1, -1, -1,
|
||||
1, -1, 6, -1, 1, -1, 0, 3, 2, 4, 15, 11, -1, 9, 5, 10, 13, -1, 12, 8, 7, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
9, 11, -1, 4, 2, -1, 0, 8, 1, 5, -1, 6, -1, 3, 7, 15, -1, 12, 10, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
-1, -1, 14, 7, 5, -1, 1, 2, 8, 9, 0, 15, 6, 4, 11, -1, 12, 3, -1, 10, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
2, 4, 3, 1, 5, 0, -1, 6, 10, 9, 7, 12, 11, -1, -1, -1, -1, 13, -1, -1, 8, -1, 15, -1, -1, -1, 14, -1, -1, -1, -1, -1,
|
||||
0, 1, 2, 3, 4, -1, -1, 5, 9, 10, 6, -1, -1, 8, 15, 11, -1, 14, -1, -1, 7, -1, 13, -1, -1, -1, 12, -1, -1, -1, -1, -1,
|
||||
2, 8, 7, 4, 3, -1, 9, -1, 6, 11, -1, 5, -1, -1, 0, -1, -1, 14, 1, 15, 10, 12, -1, -1, -1, -1, 13, -1, -1, -1, -1, -1,
|
||||
0, 3, 1, 2, 6, -1, 9, 8, 4, 12, 13, 10, -1, 11, 7, -1, -1, 15, 14, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
0, 6, 3, 4, 1, 2, -1, -1, 5, 10, 7, 9, 11, 12, -1, -1, 8, 14, -1, -1, 15, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
0, 6, 2, 5, 9, -1, -1, -1, 10, 1, 8, -1, 12, 14, 4, -1, 15, 7, -1, 13, 3, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
8, 10, 9, 15, 1, -1, 4, 0, 3, 2, -1, 6, -1, 12, 11, 13, 7, 14, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
1, 3, 6, 0, 4, 2, -1, 7, 13, 8, 9, 11, -1, -1, 15, -1, -1, -1, -1, -1, 10, 5, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
3, 0, 1, 4, -1, 2, 5, 6, 7, 8, -1, 14, -1, -1, 9, 15, -1, 12, -1, -1, -1, 10, 11, -1, -1, -1, 13, -1, -1, -1, -1, -1,
|
||||
0, 1, 3, 2, 15, -1, 12, -1, 7, 14, 4, -1, -1, 9, -1, 8, 5, 10, -1, -1, 6, -1, 13, -1, -1, -1, 11, -1, -1, -1, -1, -1,
|
||||
0, 3, 1, 2, -1, -1, 12, 6, 4, 9, 7, -1, -1, 14, 8, -1, -1, 15, 11, 13, 5, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
0, 5, 7, 2, 10, 13, -1, 6, 8, 1, 3, -1, -1, 14, 15, 11, -1, -1, -1, 12, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
0, 2, 6, 3, 7, 10, -1, 1, 9, 4, 8, -1, -1, 15, -1, 12, 5, -1, -1, -1, 11, -1, 13, -1, -1, -1, 14, -1, -1, -1, -1, -1,
|
||||
1, 3, 4, 0, 7, -1, 12, 2, 11, 8, 6, 13, -1, -1, -1, -1, -1, 5, -1, -1, 10, 15, 9, -1, -1, -1, 14, -1, -1, -1, -1, -1,
|
||||
1, 3, 5, 2, 13, 0, 9, 4, 7, 6, 8, -1, -1, 15, -1, 11, -1, -1, 10, -1, 14, -1, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
0, 2, 1, 3, -1, -1, -1, 6, -1, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
1, 11, 4, 0, 3, -1, 13, 12, 2, 7, -1, -1, 15, 10, 5, 8, 14, -1, -1, -1, -1, -1, 9, -1, -1, -1, 6, -1, -1, -1, -1, -1,
|
||||
0, 9, 2, 14, 15, 4, 1, 13, 3, 5, -1, -1, 10, -1, -1, -1, -1, 6, 12, -1, 7, -1, 8, -1, -1, -1, 11, -1, -1, -1, -1, -1,
|
||||
-1, 2, 14, -1, 1, 5, 8, 7, 4, 12, -1, 6, 9, 11, 13, 3, 10, 15, -1, -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
0, 1, 3, 2, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
4, 3, 1, 5, -1, -1, -1, 0, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
2, 8, 4, 1, -1, 0, -1, 6, -1, -1, 5, -1, 7, -1, -1, -1, -1, -1, -1, -1, 10, -1, -1, 9, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
12, 5, -1, -1, 1, -1, -1, 7, 0, 3, -1, 2, -1, 4, 6, -1, -1, -1, -1, 8, -1, -1, 15, -1, 13, 9, -1, -1, -1, -1, -1, 11,
|
||||
1, 3, 2, 4, -1, -1, -1, 5, -1, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, 8, -1, -1,
|
||||
5, 3, 4, 12, 1, 6, -1, -1, -1, -1, 8, 2, -1, -1, -1, -1, 0, 9, -1, -1, 11, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
||||
-1, -1, -1, -1, 0, -1, 1, 12, 3, -1, -1, -1, -1, 5, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, 4, -1, -1, 6, -1, 10,
|
||||
2, 3, 1, 4, -1, 0, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 7, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1,
|
||||
5, 1, 3, 0, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, 9, -1, -1, 6, -1, 7,
|
||||
},
|
||||
successors_reversed = {
|
||||
's', 't', 'c', 'l', 'm', 'a', 'd', 'r', 'v', 'T', 'A', 'L', 'e', 'M', 'Y', '-',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'-', 't', 'a', 'b', 's', 'h', 'c', 'r', 'n', 'w', 'p', 'm', 'l', 'd', 'i', 'f',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'u', 'e', 'i', 'a', 'o', 'r', 'y', 'l', 'I', 'E', 'R', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'e', 'a', 'o', 'i', 'u', 'A', 'y', 'E', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
't', 'n', 'f', 's', '\'', 'm', 'I', 'N', 'A', 'E', 'L', 'Z', 'r', 'V', 'R', 'C',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'o', 'a', 'y', 'i', 'u', 'e', 'I', 'L', 'D', '\'', 'E', 'Y', '\x00', '\x00', '\x00', '\x00',
|
||||
'r', 'i', 'y', 'a', 'e', 'o', 'u', 'Y', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'h', 'o', 'e', 'E', 'i', 'u', 'r', 'w', 'a', 'H', 'y', 'R', 'Z', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'h', 'i', 'e', 'a', 'o', 'r', 'I', 'y', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'n', 't', 's', 'r', 'l', 'd', 'i', 'y', 'v', 'm', 'b', 'c', 'g', 'p', 'k', 'u',
|
||||
'e', 'l', 'o', 'u', 'y', 'a', 'r', 'i', 's', 'j', 't', 'b', 'v', 'h', 'm', 'd',
|
||||
'o', 'e', 'h', 'a', 't', 'k', 'i', 'r', 'l', 'u', 'y', 'c', 'q', 's', '-', 'd',
|
||||
'e', 'i', 'o', 'a', 's', 'y', 'r', 'u', 'd', 'l', '-', 'g', 'n', 'v', 'm', 'f',
|
||||
'r', 'n', 'd', 's', 'a', 'l', 't', 'e', 'm', 'c', 'v', 'y', 'i', 'x', 'f', 'p',
|
||||
'o', 'e', 'r', 'a', 'i', 'f', 'u', 't', 'l', '-', 'y', 's', 'n', 'c', '\'', 'k',
|
||||
'h', 'e', 'o', 'a', 'r', 'i', 'l', 's', 'u', 'n', 'g', 'b', '-', 't', 'y', 'm',
|
||||
'e', 'a', 'i', 'o', 't', 'r', 'u', 'y', 'm', 's', 'l', 'b', '\'', '-', 'f', 'd',
|
||||
'n', 's', 't', 'm', 'o', 'l', 'c', 'd', 'r', 'e', 'g', 'a', 'f', 'v', 'z', 'b',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'e', 'n', 'i', 's', 'h', 'l', 'f', 'y', '-', 'a', 'w', '\'', 'g', 'r', 'o', 't',
|
||||
'e', 'l', 'i', 'y', 'd', 'o', 'a', 'f', 'u', 't', 's', 'k', 'w', 'v', 'm', 'p',
|
||||
'e', 'a', 'o', 'i', 'u', 'p', 'y', 's', 'b', 'm', 'f', '\'', 'n', '-', 'l', 't',
|
||||
'd', 'g', 'e', 't', 'o', 'c', 's', 'i', 'a', 'n', 'y', 'l', 'k', '\'', 'f', 'v',
|
||||
'u', 'n', 'r', 'f', 'm', 't', 'w', 'o', 's', 'l', 'v', 'd', 'p', 'k', 'i', 'c',
|
||||
'e', 'r', 'a', 'o', 'l', 'p', 'i', 't', 'u', 's', 'h', 'y', 'b', '-', '\'', 'm',
|
||||
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'e', 'i', 'o', 'a', 's', 'y', 't', 'd', 'r', 'n', 'c', 'm', 'l', 'u', 'g', 'f',
|
||||
'e', 't', 'h', 'i', 'o', 's', 'a', 'u', 'p', 'c', 'l', 'w', 'm', 'k', 'f', 'y',
|
||||
'h', 'o', 'e', 'i', 'a', 't', 'r', 'u', 'y', 'l', 's', 'w', 'c', 'f', '\'', '-',
|
||||
'r', 't', 'l', 's', 'n', 'g', 'c', 'p', 'e', 'i', 'a', 'd', 'm', 'b', 'f', 'o',
|
||||
'e', 'i', 'a', 'o', 'y', 'u', 'r', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
|
||||
'a', 'i', 'h', 'e', 'o', 'n', 'r', 's', 'l', 'd', 'k', '-', 'f', '\'', 'c', 'b',
|
||||
'p', 't', 'c', 'a', 'i', 'e', 'h', 'q', 'u', 'f', '-', 'y', 'o', '\x00', '\x00', '\x00',
|
||||
'o', 'e', 's', 't', 'i', 'd', '\'', 'l', 'b', '-', 'm', 'a', 'r', 'n', 'p', 'w',
|
||||
},
|
||||
|
||||
character_count = 32,
|
||||
successor_count = 16,
|
||||
|
||||
max_successor_n = 7,
|
||||
packs = {
|
||||
{ 0x80000000, 1, 2, { 26, 24, 24, 24, 24, 24, 24, 24 }, { 15, 3, 0, 0, 0, 0, 0, 0 }, 0xc0, 0x80 },
|
||||
{ 0xc0000000, 2, 4, { 25, 22, 19, 16, 16, 16, 16, 16 }, { 15, 7, 7, 7, 0, 0, 0, 0 }, 0xe0, 0xc0 },
|
||||
{ 0xe0000000, 4, 8, { 23, 19, 15, 11, 8, 5, 2, 0 }, { 31, 15, 15, 15, 7, 7, 7, 3 }, 0xf0, 0xe0 },
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
/*
|
||||
Copyright 2022 Jeroen van Rijn <nom@duclavier.com>.
|
||||
Made available under Odin's BSD-3 license.
|
||||
|
||||
List of contributors:
|
||||
Jeroen van Rijn: Initial implementation.
|
||||
|
||||
An implementation of [shoco](https://github.com/Ed-von-Schleck/shoco) by Christian Schramm.
|
||||
*/
|
||||
|
||||
// package shoco is an implementation of the shoco short string compressor
|
||||
package shoco
|
||||
|
||||
import "core:intrinsics"
|
||||
import "core:compress"
|
||||
|
||||
Shoco_Pack :: struct {
|
||||
word: u32,
|
||||
bytes_packed: i8,
|
||||
bytes_unpacked: i8,
|
||||
offsets: [8]u16,
|
||||
masks: [8]i16,
|
||||
header_mask: u8,
|
||||
header: u8,
|
||||
}
|
||||
|
||||
Shoco_Model :: struct {
|
||||
min_char: u8,
|
||||
max_char: u8,
|
||||
characters_by_id: []u8,
|
||||
ids_by_character: [256]i16,
|
||||
successors_by_bigram: []i8,
|
||||
successors_reversed: []u8,
|
||||
|
||||
character_count: u8,
|
||||
successor_count: u8,
|
||||
max_successor_n: i8,
|
||||
packs: []Shoco_Pack,
|
||||
}
|
||||
|
||||
compress_bound :: proc(uncompressed_size: int) -> (worst_case_compressed_size: int) {
|
||||
// Worst case compression happens when input is non-ASCII (128-255)
|
||||
// Encoded as 0x00 + the byte in question.
|
||||
return uncompressed_size * 2
|
||||
}
|
||||
|
||||
decompress_bound :: proc(compressed_size: int, model := DEFAULT_MODEL) -> (maximum_decompressed_size: int) {
|
||||
// Best case compression is 2:1
|
||||
most: f64
|
||||
for pack in model.packs {
|
||||
val := f64(compressed_size) / f64(pack.bytes_packed) * f64(pack.bytes_unpacked)
|
||||
most = max(most, val)
|
||||
}
|
||||
return int(most)
|
||||
}
|
||||
|
||||
find_best_encoding :: proc(indices: []i16, n_consecutive: i8, model := DEFAULT_MODEL) -> (res: int) {
|
||||
for p := len(model.packs); p > 0; p -= 1 {
|
||||
pack := model.packs[p - 1]
|
||||
if n_consecutive >= pack.bytes_unpacked {
|
||||
have_index := true
|
||||
for i := 0; i < int(pack.bytes_unpacked); i += 1 {
|
||||
if indices[i] > pack.masks[i] {
|
||||
have_index = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if have_index {
|
||||
return p - 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
validate_model :: proc(model: Shoco_Model) -> (int, compress.Error) {
|
||||
if len(model.characters_by_id) != int(model.character_count) {
|
||||
return 0, .Unknown_Compression_Method
|
||||
}
|
||||
|
||||
if len(model.successors_by_bigram) != int(model.character_count) * int(model.character_count) {
|
||||
return 0, .Unknown_Compression_Method
|
||||
}
|
||||
|
||||
if len(model.successors_reversed) != int(model.successor_count) * int(model.max_char - model.min_char) {
|
||||
return 0, .Unknown_Compression_Method
|
||||
}
|
||||
|
||||
// Model seems legit.
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Decompresses into provided buffer.
|
||||
decompress_slice_to_output_buffer :: proc(input: []u8, output: []u8, model := DEFAULT_MODEL) -> (size: int, err: compress.Error) {
|
||||
inp, inp_end := 0, len(input)
|
||||
out, out_end := 0, len(output)
|
||||
|
||||
validate_model(model) or_return
|
||||
|
||||
for inp < inp_end {
|
||||
val := transmute(i8)input[inp]
|
||||
mark := int(-1)
|
||||
|
||||
for val < 0 {
|
||||
val <<= 1
|
||||
mark += 1
|
||||
}
|
||||
|
||||
if mark > len(model.packs) {
|
||||
return out, .Unknown_Compression_Method
|
||||
}
|
||||
|
||||
if mark < 0 {
|
||||
if out >= out_end {
|
||||
return out, .Output_Too_Short
|
||||
}
|
||||
|
||||
// Ignore the sentinel value for non-ASCII chars
|
||||
if input[inp] == 0x00 {
|
||||
inp += 1
|
||||
if inp >= inp_end {
|
||||
return out, .Stream_Too_Short
|
||||
}
|
||||
}
|
||||
output[out] = input[inp]
|
||||
inp, out = inp + 1, out + 1
|
||||
|
||||
} else {
|
||||
pack := model.packs[mark]
|
||||
|
||||
if out + int(pack.bytes_unpacked) > out_end {
|
||||
return out, .Output_Too_Short
|
||||
} else if inp + int(pack.bytes_packed) > inp_end {
|
||||
return out, .Stream_Too_Short
|
||||
}
|
||||
|
||||
code := intrinsics.unaligned_load((^u32)(&input[inp]))
|
||||
when ODIN_ENDIAN == .Little {
|
||||
code = intrinsics.byte_swap(code)
|
||||
}
|
||||
|
||||
// Unpack the leading char
|
||||
offset := pack.offsets[0]
|
||||
mask := pack.masks[0]
|
||||
|
||||
last_chr := model.characters_by_id[(code >> offset) & u32(mask)]
|
||||
output[out] = last_chr
|
||||
|
||||
// Unpack the successor chars
|
||||
for i := 1; i < int(pack.bytes_unpacked); i += 1 {
|
||||
offset = pack.offsets[i]
|
||||
mask = pack.masks[i]
|
||||
|
||||
index_major := u32(last_chr - model.min_char) * u32(model.successor_count)
|
||||
index_minor := (code >> offset) & u32(mask)
|
||||
|
||||
last_chr = model.successors_reversed[index_major + index_minor]
|
||||
|
||||
output[out + i] = last_chr
|
||||
}
|
||||
|
||||
out += int(pack.bytes_unpacked)
|
||||
inp += int(pack.bytes_packed)
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
decompress_slice_to_string :: proc(input: []u8, model := DEFAULT_MODEL, allocator := context.allocator) -> (res: string, err: compress.Error) {
|
||||
context.allocator = allocator
|
||||
|
||||
if len(input) == 0 {
|
||||
return "", .Stream_Too_Short
|
||||
}
|
||||
|
||||
max_output_size := decompress_bound(len(input), model)
|
||||
|
||||
buf: [dynamic]u8
|
||||
if !resize(&buf, max_output_size) {
|
||||
return "", .Out_Of_Memory
|
||||
}
|
||||
|
||||
length, result := decompress_slice_to_output_buffer(input, buf[:])
|
||||
resize(&buf, length)
|
||||
return string(buf[:]), result
|
||||
}
|
||||
decompress :: proc{decompress_slice_to_output_buffer, decompress_slice_to_string}
|
||||
|
||||
compress_string_to_buffer :: proc(input: string, output: []u8, model := DEFAULT_MODEL, allocator := context.allocator) -> (size: int, err: compress.Error) {
|
||||
inp, inp_end := 0, len(input)
|
||||
out, out_end := 0, len(output)
|
||||
output := output
|
||||
|
||||
validate_model(model) or_return
|
||||
|
||||
indices := make([]i16, model.max_successor_n + 1)
|
||||
defer delete(indices)
|
||||
|
||||
last_resort := false
|
||||
|
||||
encode: for inp < inp_end {
|
||||
if last_resort {
|
||||
last_resort = false
|
||||
|
||||
if input[inp] & 0x80 == 0x80 {
|
||||
// Non-ASCII case
|
||||
if out + 2 > out_end {
|
||||
return out, .Output_Too_Short
|
||||
}
|
||||
|
||||
// Put in a sentinel byte
|
||||
output[out] = 0x00
|
||||
out += 1
|
||||
} else {
|
||||
// An ASCII byte
|
||||
if out + 1 > out_end {
|
||||
return out, .Output_Too_Short
|
||||
}
|
||||
}
|
||||
output[out] = input[inp]
|
||||
out, inp = out + 1, inp + 1
|
||||
} else {
|
||||
// Find the longest string of known successors
|
||||
indices[0] = model.ids_by_character[input[inp]]
|
||||
last_chr_index := indices[0]
|
||||
|
||||
if last_chr_index < 0 {
|
||||
last_resort = true
|
||||
continue encode
|
||||
}
|
||||
|
||||
rest := inp_end - inp
|
||||
n_consecutive: i8 = 1
|
||||
for ; n_consecutive <= model.max_successor_n; n_consecutive += 1 {
|
||||
if inp_end > 0 && int(n_consecutive) == rest {
|
||||
break
|
||||
}
|
||||
|
||||
current_index := model.ids_by_character[input[inp + int(n_consecutive)]]
|
||||
if current_index < 0 { // '\0' is always -1
|
||||
break
|
||||
}
|
||||
|
||||
successor_index := model.successors_by_bigram[last_chr_index * i16(model.character_count) + current_index]
|
||||
if successor_index < 0 {
|
||||
break
|
||||
}
|
||||
|
||||
indices[n_consecutive] = i16(successor_index)
|
||||
last_chr_index = current_index
|
||||
}
|
||||
|
||||
if n_consecutive < 2 {
|
||||
last_resort = true
|
||||
continue encode
|
||||
}
|
||||
|
||||
pack_n := find_best_encoding(indices, n_consecutive)
|
||||
if pack_n >= 0 {
|
||||
if out + int(model.packs[pack_n].bytes_packed) > out_end {
|
||||
return out, .Output_Too_Short
|
||||
}
|
||||
|
||||
pack := model.packs[pack_n]
|
||||
code := pack.word
|
||||
|
||||
for i := 0; i < int(pack.bytes_unpacked); i += 1 {
|
||||
code |= u32(indices[i]) << pack.offsets[i]
|
||||
}
|
||||
|
||||
// In the little-endian world, we need to swap what's in the register to match the memory representation.
|
||||
when ODIN_ENDIAN == .Little {
|
||||
code = intrinsics.byte_swap(code)
|
||||
}
|
||||
out_ptr := raw_data(output[out:])
|
||||
|
||||
switch pack.bytes_packed {
|
||||
case 4:
|
||||
intrinsics.unaligned_store(transmute(^u32)out_ptr, code)
|
||||
case 2:
|
||||
intrinsics.unaligned_store(transmute(^u16)out_ptr, u16(code))
|
||||
case 1:
|
||||
intrinsics.unaligned_store(transmute(^u8)out_ptr, u8(code))
|
||||
case:
|
||||
return out, .Unknown_Compression_Method
|
||||
}
|
||||
|
||||
out += int(pack.bytes_packed)
|
||||
inp += int(pack.bytes_unpacked)
|
||||
} else {
|
||||
last_resort = true
|
||||
continue encode
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
compress_string :: proc(input: string, model := DEFAULT_MODEL, allocator := context.allocator) -> (output: []u8, err: compress.Error) {
|
||||
context.allocator = allocator
|
||||
|
||||
if len(input) == 0 {
|
||||
return {}, .Stream_Too_Short
|
||||
}
|
||||
|
||||
max_output_size := compress_bound(len(input))
|
||||
|
||||
buf: [dynamic]u8
|
||||
if !resize(&buf, max_output_size) {
|
||||
return {}, .Out_Of_Memory
|
||||
}
|
||||
|
||||
length, result := compress_string_to_buffer(input, buf[:])
|
||||
resize(&buf, length)
|
||||
return buf[:length], result
|
||||
}
|
||||
compress :: proc{compress_string_to_buffer, compress_string}
|
||||
@@ -0,0 +1,173 @@
|
||||
package container_intrusive_list
|
||||
|
||||
import "core:intrinsics"
|
||||
|
||||
// An intrusive doubly-linked list
|
||||
//
|
||||
// As this is an intrusive container, a `Node` must be embedded in your own
|
||||
// structure which is conventionally called a "link". The use of `push_front`
|
||||
// and `push_back` take the address of this node. Retrieving the data
|
||||
// associated with the node requires finding the relative offset of the node
|
||||
// of the parent structure. The parent type and field name are given to
|
||||
// `iterator_*` procedures, or to the built-in `container_of` procedure.
|
||||
//
|
||||
// This data structure is two-pointers in size:
|
||||
// 8 bytes on 32-bit platforms and 16 bytes on 64-bit platforms
|
||||
List :: struct {
|
||||
head: ^Node,
|
||||
tail: ^Node,
|
||||
}
|
||||
|
||||
|
||||
Node :: struct {
|
||||
next, prev: ^Node,
|
||||
}
|
||||
|
||||
push_front :: proc(list: ^List, node: ^Node) {
|
||||
if list.head != nil {
|
||||
list.head.prev = node
|
||||
node.prev, node.next = nil, list.head
|
||||
list.head = node
|
||||
} else {
|
||||
list.head, list.tail = node, node
|
||||
node.prev, node.next = nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
push_back :: proc(list: ^List, node: ^Node) {
|
||||
if list.tail != nil {
|
||||
list.tail.next = node
|
||||
node.prev, node.next = list.tail, nil
|
||||
list.tail = node
|
||||
} else {
|
||||
list.head, list.tail = node, node
|
||||
node.prev, node.next = nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
remove :: proc(list: ^List, node: ^Node) {
|
||||
if node != nil {
|
||||
if node.next != nil {
|
||||
node.next.prev = node.prev
|
||||
}
|
||||
if node.prev != nil {
|
||||
node.prev.next = node.next
|
||||
}
|
||||
if list.head == node {
|
||||
list.head = node.next
|
||||
}
|
||||
if list.tail == node {
|
||||
list.tail = node.prev
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
remove_by_proc :: proc(list: ^List, to_erase: proc(^Node) -> bool) {
|
||||
for node := list.head; node != nil; {
|
||||
next := node.next
|
||||
if to_erase(node) {
|
||||
if node.next != nil {
|
||||
node.next.prev = node.prev
|
||||
}
|
||||
if node.prev != nil {
|
||||
node.prev.next = node.next
|
||||
}
|
||||
if list.head == node {
|
||||
list.head = node.next
|
||||
}
|
||||
if list.tail == node {
|
||||
list.tail = node.prev
|
||||
}
|
||||
}
|
||||
node = next
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
is_empty :: proc(list: ^List) -> bool {
|
||||
return list.head == nil
|
||||
}
|
||||
|
||||
pop_front :: proc(list: ^List) -> ^Node {
|
||||
link := list.head
|
||||
if link == nil {
|
||||
return nil
|
||||
}
|
||||
if link.next != nil {
|
||||
link.next.prev = link.prev
|
||||
}
|
||||
if link.prev != nil {
|
||||
link.prev.next = link.next
|
||||
}
|
||||
if link == list.head {
|
||||
list.head = link.next
|
||||
}
|
||||
if link == list.tail {
|
||||
list.tail = link.prev
|
||||
}
|
||||
return link
|
||||
|
||||
}
|
||||
pop_back :: proc(list: ^List) -> ^Node {
|
||||
link := list.tail
|
||||
if link == nil {
|
||||
return nil
|
||||
}
|
||||
if link.next != nil {
|
||||
link.next.prev = link.prev
|
||||
}
|
||||
if link.prev != nil {
|
||||
link.prev.next = link.next
|
||||
}
|
||||
if link == list.head {
|
||||
list.head = link.next
|
||||
}
|
||||
if link == list.tail {
|
||||
list.tail = link.prev
|
||||
}
|
||||
return link
|
||||
}
|
||||
|
||||
|
||||
Iterator :: struct($T: typeid) {
|
||||
curr: ^Node,
|
||||
offset: uintptr,
|
||||
}
|
||||
|
||||
iterator_head :: proc(list: List, $T: typeid, $field_name: string) -> Iterator(T)
|
||||
where intrinsics.type_has_field(T, field_name),
|
||||
intrinsics.type_field_type(T, field_name) == Node {
|
||||
return {list.head, offset_of_by_string(T, field_name)}
|
||||
}
|
||||
|
||||
iterator_tail :: proc(list: List, $T: typeid, $field_name: string) -> Iterator(T)
|
||||
where intrinsics.type_has_field(T, field_name),
|
||||
intrinsics.type_field_type(T, field_name) == Node {
|
||||
return {list.tail, offset_of_by_string(T, field_name)}
|
||||
}
|
||||
|
||||
iterator_from_node :: proc(node: ^Node, $T: typeid, $field_name: string) -> Iterator(T)
|
||||
where intrinsics.type_has_field(T, field_name),
|
||||
intrinsics.type_field_type(T, field_name) == Node {
|
||||
return {node, offset_of_by_string(T, field_name)}
|
||||
}
|
||||
|
||||
iterate_next :: proc(it: ^Iterator($T)) -> (ptr: ^T, ok: bool) {
|
||||
node := it.curr
|
||||
if node == nil {
|
||||
return nil, false
|
||||
}
|
||||
it.curr = node.next
|
||||
|
||||
return (^T)(uintptr(node) - it.offset), true
|
||||
}
|
||||
|
||||
iterate_prev :: proc(it: ^Iterator($T)) -> (ptr: ^T, ok: bool) {
|
||||
node := it.curr
|
||||
if node == nil {
|
||||
return nil, false
|
||||
}
|
||||
it.curr = node.prev
|
||||
|
||||
return (^T)(uintptr(node) - it.offset), true
|
||||
}
|
||||
@@ -65,20 +65,22 @@ set :: proc(c: ^$C/Cache($Key, $Value), key: Key, value: Value) -> runtime.Alloc
|
||||
return nil
|
||||
}
|
||||
|
||||
e := new(Node(Key, Value), c.node_allocator) or_return
|
||||
e.key = key
|
||||
e.value = value
|
||||
|
||||
e : ^Node(Key, Value) = nil
|
||||
assert(c.count <= c.capacity)
|
||||
if c.count == c.capacity {
|
||||
_remove_node(c, c.tail)
|
||||
e = c.tail
|
||||
_remove_node(c, e)
|
||||
}
|
||||
else {
|
||||
c.count += 1
|
||||
e = new(Node(Key, Value), c.node_allocator) or_return
|
||||
}
|
||||
_push_front_node(c, e)
|
||||
|
||||
e.key = key
|
||||
e.value = value
|
||||
_push_front_node(c, e)
|
||||
c.entries[key] = e
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -128,6 +130,7 @@ remove :: proc(c: ^$C/Cache($Key, $Value), key: Key) -> bool {
|
||||
return false
|
||||
}
|
||||
_remove_node(c, e)
|
||||
free(node, c.node_allocator)
|
||||
c.count -= 1
|
||||
return true
|
||||
}
|
||||
@@ -153,9 +156,6 @@ _remove_node :: proc(c: ^$C/Cache($Key, $Value), node: ^Node(Key, Value)) {
|
||||
delete_key(&c.entries, node.key)
|
||||
|
||||
_call_on_remove(c, node)
|
||||
|
||||
free(node, c.node_allocator)
|
||||
|
||||
}
|
||||
|
||||
@(private)
|
||||
|
||||
@@ -86,7 +86,7 @@ pop_back_safe :: proc(a: ^$A/Small_Array($N, $T)) -> (item: T, ok: bool) {
|
||||
return
|
||||
}
|
||||
|
||||
pop_front_safe :: proc(a: ^$A/Small_Array($N, $T)) -> (T, bool) {
|
||||
pop_front_safe :: proc(a: ^$A/Small_Array($N, $T)) -> (item: T, ok: bool) {
|
||||
if N > 0 && a.len > 0 {
|
||||
item = a.data[0]
|
||||
s := slice(a)
|
||||
@@ -114,4 +114,4 @@ push_back_elems :: proc(a: ^$A/Small_Array($N, $T), items: ..T) {
|
||||
append_elem :: push_back
|
||||
append_elems :: push_back_elems
|
||||
push :: proc{push_back, push_back_elems}
|
||||
append :: proc{push_back, push_back_elems}
|
||||
append :: proc{push_back, push_back_elems}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# License
|
||||
|
||||
By obtaining, using and/or copying this work, you (the licensee) agree that you have read, understood, and will comply with the following terms and conditions.
|
||||
|
||||
Permission to copy, modify, and distribute this software and its documentation, with or without modification, for any purpose and without fee or royalty is hereby granted, provided that you include the following on ALL copies of the software and documentation or portions thereof, including modifications:
|
||||
|
||||
The full text of this NOTICE in a location viewable to users of the redistributed or derivative work.
|
||||
Any pre-existing intellectual property disclaimers, notices, or terms and conditions. If none exist, the W3C Software Short Notice should be included (hypertext is preferred, text is permitted) within the body of any redistributed or derivative code.
|
||||
|
||||
Notice of any changes or modifications to the files, including the date changes were made. (We recommend you provide URIs to the location from which the code is derived.)
|
||||
|
||||
# Disclaimers
|
||||
|
||||
THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE OR DOCUMENTATION WILL NOT INFRINGE ANY THIRD PARTY PATENTS, COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS.
|
||||
|
||||
COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE SOFTWARE OR DOCUMENTATION.
|
||||
|
||||
The name and trademarks of copyright holders may NOT be used in advertising or publicity pertaining to the software without specific, written prior permission. Title to copyright in this software and any associated documentation will at all times remain with copyright holders.
|
||||
|
||||
# Notes
|
||||
This version: http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231
|
||||
@@ -0,0 +1,374 @@
|
||||
package unicode_entity
|
||||
/*
|
||||
A unicode entity encoder/decoder
|
||||
|
||||
Copyright 2021 Jeroen van Rijn <nom@duclavier.com>.
|
||||
Made available under Odin's BSD-3 license.
|
||||
|
||||
This code has several procedures to map unicode runes to/from different textual encodings.
|
||||
- SGML/XML/HTML entity
|
||||
-- &#<decimal>;
|
||||
-- &#x<hexadecimal>;
|
||||
-- &<entity name>; (If the lookup tables are compiled in).
|
||||
Reference: https://www.w3.org/2003/entities/2007xml/unicode.xml
|
||||
|
||||
- URL encode / decode %hex entity
|
||||
Reference: https://datatracker.ietf.org/doc/html/rfc3986/#section-2.1
|
||||
|
||||
List of contributors:
|
||||
Jeroen van Rijn: Initial implementation.
|
||||
*/
|
||||
|
||||
import "core:unicode/utf8"
|
||||
import "core:unicode"
|
||||
import "core:strings"
|
||||
|
||||
MAX_RUNE_CODEPOINT :: int(unicode.MAX_RUNE)
|
||||
|
||||
write_rune :: strings.write_rune_builder
|
||||
write_string :: strings.write_string_builder
|
||||
|
||||
Error :: enum u8 {
|
||||
None = 0,
|
||||
Tokenizer_Is_Nil,
|
||||
|
||||
Illegal_NUL_Character,
|
||||
Illegal_UTF_Encoding,
|
||||
Illegal_BOM,
|
||||
|
||||
CDATA_Not_Terminated,
|
||||
Comment_Not_Terminated,
|
||||
Invalid_Entity_Encoding,
|
||||
}
|
||||
|
||||
Tokenizer :: struct {
|
||||
r: rune,
|
||||
w: int,
|
||||
|
||||
src: string,
|
||||
offset: int,
|
||||
read_offset: int,
|
||||
}
|
||||
|
||||
CDATA_START :: "<![CDATA["
|
||||
CDATA_END :: "]]>"
|
||||
|
||||
COMMENT_START :: "<!--"
|
||||
COMMENT_END :: "-->"
|
||||
|
||||
/*
|
||||
Default: CDATA and comments are passed through unchanged.
|
||||
*/
|
||||
XML_Decode_Option :: enum u8 {
|
||||
/*
|
||||
Do not decode & entities. It decodes by default.
|
||||
If given, overrides `Decode_CDATA`.
|
||||
*/
|
||||
No_Entity_Decode,
|
||||
|
||||
/*
|
||||
CDATA is unboxed.
|
||||
*/
|
||||
Unbox_CDATA,
|
||||
|
||||
/*
|
||||
Unboxed CDATA is decoded as well.
|
||||
Ignored if `.Unbox_CDATA` is not given.
|
||||
*/
|
||||
Decode_CDATA,
|
||||
|
||||
/*
|
||||
Comments are stripped.
|
||||
*/
|
||||
Comment_Strip,
|
||||
}
|
||||
XML_Decode_Options :: bit_set[XML_Decode_Option; u8]
|
||||
|
||||
/*
|
||||
Decode a string that may include SGML/XML/HTML entities.
|
||||
The caller has to free the result.
|
||||
*/
|
||||
decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator := context.allocator) -> (decoded: string, err: Error) {
|
||||
context.allocator = allocator
|
||||
|
||||
l := len(input)
|
||||
if l == 0 { return "", .None }
|
||||
|
||||
builder := strings.make_builder()
|
||||
defer strings.destroy_builder(&builder)
|
||||
|
||||
t := Tokenizer{src=input}
|
||||
in_data := false
|
||||
|
||||
loop: for {
|
||||
advance(&t) or_return
|
||||
if t.r < 0 { break loop }
|
||||
|
||||
/*
|
||||
Below here we're never inside a CDATA tag.
|
||||
At most we'll see the start of one, but that doesn't affect the logic.
|
||||
*/
|
||||
switch t.r {
|
||||
case '<':
|
||||
/*
|
||||
Might be the start of a CDATA tag or comment.
|
||||
|
||||
We don't need to check if we need to write a `<`, because if it isn't CDATA or a comment,
|
||||
it couldn't have been part of an XML tag body to be decoded here.
|
||||
|
||||
Keep in mind that we could already *be* inside a CDATA tag.
|
||||
If so, write `>` as a literal and continue.
|
||||
*/
|
||||
if in_data {
|
||||
write_rune(&builder, '<')
|
||||
continue
|
||||
}
|
||||
in_data = _handle_xml_special(&t, &builder, options) or_return
|
||||
|
||||
case ']':
|
||||
/*
|
||||
If we're unboxing _and_ decoding CDATA, we'll have to check for the end tag.
|
||||
*/
|
||||
if in_data {
|
||||
if t.read_offset + len(CDATA_END) < len(t.src) {
|
||||
if string(t.src[t.offset:][:len(CDATA_END)]) == CDATA_END {
|
||||
in_data = false
|
||||
t.read_offset += len(CDATA_END) - 1
|
||||
}
|
||||
}
|
||||
continue
|
||||
} else {
|
||||
write_rune(&builder, ']')
|
||||
}
|
||||
|
||||
case:
|
||||
if in_data && .Decode_CDATA not_in options {
|
||||
/*
|
||||
Unboxed, but undecoded.
|
||||
*/
|
||||
write_rune(&builder, t.r)
|
||||
continue
|
||||
}
|
||||
|
||||
if t.r == '&' {
|
||||
if entity, entity_err := _extract_xml_entity(&t); entity_err != .None {
|
||||
/*
|
||||
We read to the end of the string without closing the entity.
|
||||
Pass through as-is.
|
||||
*/
|
||||
write_string(&builder, entity)
|
||||
} else {
|
||||
|
||||
if .No_Entity_Decode not_in options {
|
||||
if decoded, ok := xml_decode_entity(entity); ok {
|
||||
write_rune(&builder, decoded)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Literal passthrough because the decode failed or we want entities not decoded.
|
||||
*/
|
||||
write_string(&builder, "&")
|
||||
write_string(&builder, entity)
|
||||
write_string(&builder, ";")
|
||||
}
|
||||
} else {
|
||||
write_rune(&builder, t.r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return strings.clone(strings.to_string(builder), allocator), err
|
||||
}
|
||||
|
||||
advance :: proc(t: ^Tokenizer) -> (err: Error) {
|
||||
if t == nil { return .Tokenizer_Is_Nil }
|
||||
using t
|
||||
|
||||
#no_bounds_check {
|
||||
if read_offset < len(src) {
|
||||
offset = read_offset
|
||||
r, w = rune(src[read_offset]), 1
|
||||
switch {
|
||||
case r == 0:
|
||||
return .Illegal_NUL_Character
|
||||
case r >= utf8.RUNE_SELF:
|
||||
r, w = utf8.decode_rune_in_string(src[read_offset:])
|
||||
if r == utf8.RUNE_ERROR && w == 1 {
|
||||
return .Illegal_UTF_Encoding
|
||||
} else if r == utf8.RUNE_BOM && offset > 0 {
|
||||
return .Illegal_BOM
|
||||
}
|
||||
}
|
||||
read_offset += w
|
||||
return .None
|
||||
} else {
|
||||
offset = len(src)
|
||||
r = -1
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
xml_decode_entity :: proc(entity: string) -> (decoded: rune, ok: bool) {
|
||||
entity := entity
|
||||
if len(entity) == 0 { return -1, false }
|
||||
|
||||
switch entity[0] {
|
||||
case '#':
|
||||
base := 10
|
||||
val := 0
|
||||
entity = entity[1:]
|
||||
|
||||
if len(entity) == 0 { return -1, false }
|
||||
|
||||
if entity[0] == 'x' || entity[0] == 'X' {
|
||||
base = 16
|
||||
entity = entity[1:]
|
||||
}
|
||||
|
||||
for len(entity) > 0 {
|
||||
r := entity[0]
|
||||
switch r {
|
||||
case '0'..'9':
|
||||
val *= base
|
||||
val += int(r - '0')
|
||||
|
||||
case 'a'..'f':
|
||||
if base == 10 { return -1, false }
|
||||
val *= base
|
||||
val += int(r - 'a' + 10)
|
||||
|
||||
case 'A'..'F':
|
||||
if base == 10 { return -1, false }
|
||||
val *= base
|
||||
val += int(r - 'A' + 10)
|
||||
|
||||
case:
|
||||
return -1, false
|
||||
}
|
||||
|
||||
if val > MAX_RUNE_CODEPOINT { return -1, false }
|
||||
entity = entity[1:]
|
||||
}
|
||||
return rune(val), true
|
||||
|
||||
case:
|
||||
/*
|
||||
Named entity.
|
||||
*/
|
||||
return named_xml_entity_to_rune(entity)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Private XML helper to extract `&<stuff>;` entity.
|
||||
*/
|
||||
@(private="file")
|
||||
_extract_xml_entity :: proc(t: ^Tokenizer) -> (entity: string, err: Error) {
|
||||
assert(t != nil && t.r == '&')
|
||||
|
||||
/*
|
||||
All of these would be in the ASCII range.
|
||||
Even if one is not, it doesn't matter. All characters we need to compare to extract are.
|
||||
*/
|
||||
using t
|
||||
|
||||
length := len(t.src)
|
||||
found := false
|
||||
|
||||
#no_bounds_check {
|
||||
for read_offset < length {
|
||||
if src[read_offset] == ';' {
|
||||
found = true
|
||||
read_offset += 1
|
||||
break
|
||||
}
|
||||
read_offset += 1
|
||||
}
|
||||
}
|
||||
|
||||
if found {
|
||||
return string(src[offset + 1 : read_offset - 1]), .None
|
||||
}
|
||||
return string(src[offset : read_offset]), .Invalid_Entity_Encoding
|
||||
}
|
||||
|
||||
/*
|
||||
Private XML helper for CDATA and comments.
|
||||
*/
|
||||
@(private="file")
|
||||
_handle_xml_special :: proc(t: ^Tokenizer, builder: ^strings.Builder, options: XML_Decode_Options) -> (in_data: bool, err: Error) {
|
||||
assert(t != nil && t.r == '<')
|
||||
if t.read_offset + len(CDATA_START) >= len(t.src) { return false, .None }
|
||||
|
||||
if string(t.src[t.offset:][:len(CDATA_START)]) == CDATA_START {
|
||||
t.read_offset += len(CDATA_START) - 1
|
||||
|
||||
if .Unbox_CDATA in options && .Decode_CDATA in options {
|
||||
/*
|
||||
We're unboxing _and_ decoding CDATA
|
||||
*/
|
||||
return true, .None
|
||||
}
|
||||
|
||||
/*
|
||||
CDATA is passed through.
|
||||
*/
|
||||
offset := t.offset
|
||||
|
||||
/*
|
||||
Scan until end of CDATA.
|
||||
*/
|
||||
for {
|
||||
advance(t) or_return
|
||||
if t.r < 0 { return true, .CDATA_Not_Terminated }
|
||||
|
||||
if t.read_offset + len(CDATA_END) < len(t.src) {
|
||||
if string(t.src[t.offset:][:len(CDATA_END)]) == CDATA_END {
|
||||
t.read_offset += len(CDATA_END) - 1
|
||||
|
||||
cdata := string(t.src[offset : t.read_offset])
|
||||
|
||||
if .Unbox_CDATA in options {
|
||||
cdata = cdata[len(CDATA_START):]
|
||||
cdata = cdata[:len(cdata) - len(CDATA_END)]
|
||||
}
|
||||
|
||||
write_string(builder, cdata)
|
||||
return false, .None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} else if string(t.src[t.offset:][:len(COMMENT_START)]) == COMMENT_START {
|
||||
t.read_offset += len(COMMENT_START)
|
||||
/*
|
||||
Comment is passed through by default.
|
||||
*/
|
||||
offset := t.offset
|
||||
|
||||
/*
|
||||
Scan until end of Comment.
|
||||
*/
|
||||
for {
|
||||
advance(t) or_return
|
||||
if t.r < 0 { return true, .Comment_Not_Terminated }
|
||||
|
||||
if t.read_offset + len(COMMENT_END) < len(t.src) {
|
||||
if string(t.src[t.offset:][:len(COMMENT_END)]) == COMMENT_END {
|
||||
t.read_offset += len(COMMENT_END) - 1
|
||||
|
||||
if .Comment_Strip not_in options {
|
||||
comment := string(t.src[offset : t.read_offset])
|
||||
write_string(builder, comment)
|
||||
}
|
||||
return false, .None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return false, .None
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package unicode_entity_example
|
||||
|
||||
import "core:encoding/xml"
|
||||
import "core:strings"
|
||||
import "core:mem"
|
||||
import "core:fmt"
|
||||
import "core:time"
|
||||
|
||||
doc_print :: proc(doc: ^xml.Document) {
|
||||
buf: strings.Builder
|
||||
defer strings.destroy_builder(&buf)
|
||||
w := strings.to_writer(&buf)
|
||||
|
||||
xml.print(w, doc)
|
||||
fmt.println(strings.to_string(buf))
|
||||
}
|
||||
|
||||
_entities :: proc() {
|
||||
doc: ^xml.Document
|
||||
err: xml.Error
|
||||
|
||||
DOC :: #load("../../../../tests/core/assets/XML/unicode.xml")
|
||||
|
||||
OPTIONS :: xml.Options{
|
||||
flags = {
|
||||
.Ignore_Unsupported, .Intern_Comments,
|
||||
},
|
||||
expected_doctype = "",
|
||||
}
|
||||
|
||||
parse_duration: time.Duration
|
||||
|
||||
{
|
||||
time.SCOPED_TICK_DURATION(&parse_duration)
|
||||
doc, err = xml.parse(DOC, OPTIONS)
|
||||
}
|
||||
defer xml.destroy(doc)
|
||||
|
||||
doc_print(doc)
|
||||
|
||||
ms := time.duration_milliseconds(parse_duration)
|
||||
|
||||
speed := (f64(1000.0) / ms) * f64(len(DOC)) / 1_024.0 / 1_024.0
|
||||
|
||||
fmt.printf("Parse time: %.2f ms (%.2f MiB/s).\n", ms, speed)
|
||||
fmt.printf("Error: %v\n", err)
|
||||
}
|
||||
|
||||
_main :: proc() {
|
||||
using fmt
|
||||
|
||||
options := xml.Options{ flags = { .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, .Decode_SGML_Entities }}
|
||||
|
||||
doc, _ := xml.parse(#load("test.html"), options)
|
||||
|
||||
defer xml.destroy(doc)
|
||||
doc_print(doc)
|
||||
}
|
||||
|
||||
main :: proc() {
|
||||
using fmt
|
||||
|
||||
track: mem.Tracking_Allocator
|
||||
mem.tracking_allocator_init(&track, context.allocator)
|
||||
context.allocator = mem.tracking_allocator(&track)
|
||||
|
||||
// _main()
|
||||
_entities()
|
||||
|
||||
if len(track.allocation_map) > 0 {
|
||||
println()
|
||||
for _, v in track.allocation_map {
|
||||
printf("%v Leaked %v bytes.\n", v.location, v.size)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
<html>
|
||||
<head>
|
||||
<title>Entity Reference Test</title>
|
||||
<style>
|
||||
body {
|
||||
background: #000; color: #eee;
|
||||
width: 40%;
|
||||
margin-left: auto;
|
||||
margin-right: auto;
|
||||
font-size: 14pt;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Entity Reference Test</h1>
|
||||
<div id="test_cdata_in_comment" foo="">
|
||||
Foozle]! © <!-- <![CDATA[ ® ]]> -->42&;1234&
|
||||
</div>
|
||||
<!-- EXPECTED: Foozle]! © 42&;1234& -->
|
||||
<div id="test_cdata_unwrap_and_passthrough">
|
||||
Foozle]! © <![CDATA[BOX ® /BOX]]>42&;1234&
|
||||
</div>
|
||||
<!-- EXPECTED: Foozle]! © BOX ® /BOX42&;1234& -->
|
||||
<div>
|
||||
| | | fj ` \ ® ϱ ∳ ⁏
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -354,6 +354,12 @@ unquote_string :: proc(token: Token, spec: Specification, allocator := context.a
|
||||
|
||||
b := bytes_make(len(s) + 2*utf8.UTF_MAX, 1, allocator) or_return
|
||||
w := copy(b, s[0:i])
|
||||
|
||||
if len(b) == 0 && allocator.data == nil {
|
||||
// `unmarshal_count_array` calls us with a nil allocator
|
||||
return string(b[:w]), nil
|
||||
}
|
||||
|
||||
loop: for i < len(s) {
|
||||
c := s[i]
|
||||
switch {
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
/*
|
||||
An XML 1.0 / 1.1 parser
|
||||
|
||||
Copyright 2021-2022 Jeroen van Rijn <nom@duclavier.com>.
|
||||
Made available under Odin's BSD-3 license.
|
||||
|
||||
A from-scratch XML implementation, loosely modeled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816).
|
||||
|
||||
List of contributors:
|
||||
Jeroen van Rijn: Initial implementation.
|
||||
*/
|
||||
package xml
|
||||
|
||||
import "core:io"
|
||||
import "core:fmt"
|
||||
|
||||
/*
|
||||
Just for debug purposes.
|
||||
*/
|
||||
print :: proc(writer: io.Writer, doc: ^Document) -> (written: int, err: io.Error) {
|
||||
if doc == nil { return }
|
||||
using fmt
|
||||
|
||||
written += wprintf(writer, "[XML Prolog]\n")
|
||||
|
||||
for attr in doc.prolog {
|
||||
written += wprintf(writer, "\t%v: %v\n", attr.key, attr.val)
|
||||
}
|
||||
|
||||
written += wprintf(writer, "[Encoding] %v\n", doc.encoding)
|
||||
|
||||
if len(doc.doctype.ident) > 0 {
|
||||
written += wprintf(writer, "[DOCTYPE] %v\n", doc.doctype.ident)
|
||||
|
||||
if len(doc.doctype.rest) > 0 {
|
||||
wprintf(writer, "\t%v\n", doc.doctype.rest)
|
||||
}
|
||||
}
|
||||
|
||||
for comment in doc.comments {
|
||||
written += wprintf(writer, "[Pre-root comment] %v\n", comment)
|
||||
}
|
||||
|
||||
if len(doc.elements) > 0 {
|
||||
wprintln(writer, " --- ")
|
||||
print_element(writer, doc, 0)
|
||||
wprintln(writer, " --- ")
|
||||
}
|
||||
|
||||
return written, .None
|
||||
}
|
||||
|
||||
print_element :: proc(writer: io.Writer, doc: ^Document, element_id: Element_ID, indent := 0) -> (written: int, err: io.Error) {
|
||||
using fmt
|
||||
|
||||
tab :: proc(writer: io.Writer, indent: int) {
|
||||
for _ in 0..=indent {
|
||||
wprintf(writer, "\t")
|
||||
}
|
||||
}
|
||||
|
||||
tab(writer, indent)
|
||||
|
||||
element := doc.elements[element_id]
|
||||
|
||||
if element.kind == .Element {
|
||||
wprintf(writer, "<%v>\n", element.ident)
|
||||
if len(element.value) > 0 {
|
||||
tab(writer, indent + 1)
|
||||
wprintf(writer, "[Value] %v\n", element.value)
|
||||
}
|
||||
|
||||
for attr in element.attribs {
|
||||
tab(writer, indent + 1)
|
||||
wprintf(writer, "[Attr] %v: %v\n", attr.key, attr.val)
|
||||
}
|
||||
|
||||
for child in element.children {
|
||||
print_element(writer, doc, child, indent + 1)
|
||||
}
|
||||
} else if element.kind == .Comment {
|
||||
wprintf(writer, "[COMMENT] %v\n", element.value)
|
||||
}
|
||||
|
||||
return written, .None
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package xml_example
|
||||
|
||||
import "core:encoding/xml"
|
||||
import "core:mem"
|
||||
import "core:fmt"
|
||||
import "core:time"
|
||||
import "core:strings"
|
||||
import "core:hash"
|
||||
|
||||
N :: 1
|
||||
|
||||
example :: proc() {
|
||||
using fmt
|
||||
|
||||
docs: [N]^xml.Document
|
||||
errs: [N]xml.Error
|
||||
times: [N]time.Duration
|
||||
|
||||
defer for round in 0..<N {
|
||||
xml.destroy(docs[round])
|
||||
}
|
||||
|
||||
DOC :: #load("../../../../tests/core/assets/XML/unicode.xml")
|
||||
input := DOC
|
||||
|
||||
for round in 0..<N {
|
||||
start := time.tick_now()
|
||||
|
||||
docs[round], errs[round] = xml.parse(input, xml.Options{
|
||||
flags={.Ignore_Unsupported},
|
||||
expected_doctype = "",
|
||||
})
|
||||
|
||||
end := time.tick_now()
|
||||
times[round] = time.tick_diff(start, end)
|
||||
}
|
||||
|
||||
fastest := time.Duration(max(i64))
|
||||
slowest := time.Duration(0)
|
||||
total := time.Duration(0)
|
||||
|
||||
for round in 0..<N {
|
||||
fastest = min(fastest, times[round])
|
||||
slowest = max(slowest, times[round])
|
||||
total += times[round]
|
||||
}
|
||||
|
||||
fastest_ms := time.duration_milliseconds(fastest)
|
||||
slowest_ms := time.duration_milliseconds(slowest)
|
||||
average_ms := time.duration_milliseconds(time.Duration(f64(total) / f64(N)))
|
||||
|
||||
fastest_speed := (f64(1000.0) / fastest_ms) * f64(len(DOC)) / 1_024.0 / 1_024.0
|
||||
slowest_speed := (f64(1000.0) / slowest_ms) * f64(len(DOC)) / 1_024.0 / 1_024.0
|
||||
average_speed := (f64(1000.0) / average_ms) * f64(len(DOC)) / 1_024.0 / 1_024.0
|
||||
|
||||
fmt.printf("N = %v\n", N)
|
||||
fmt.printf("[Fastest]: %v bytes in %.2f ms (%.2f MiB/s).\n", len(input), fastest_ms, fastest_speed)
|
||||
fmt.printf("[Slowest]: %v bytes in %.2f ms (%.2f MiB/s).\n", len(input), slowest_ms, slowest_speed)
|
||||
fmt.printf("[Average]: %v bytes in %.2f ms (%.2f MiB/s).\n", len(input), average_ms, average_speed)
|
||||
|
||||
if errs[0] != .None {
|
||||
printf("Load/Parse error: %v\n", errs[0])
|
||||
if errs[0] == .File_Error {
|
||||
println("\"unicode.xml\" not found. Did you run \"tests\\download_assets.py\"?")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
charlist, charlist_ok := xml.find_child_by_ident(docs[0], 0, "charlist")
|
||||
if !charlist_ok {
|
||||
eprintln("Could not locate top-level `<charlist>` tag.")
|
||||
return
|
||||
}
|
||||
|
||||
printf("Found `<charlist>` with %v children, %v elements total\n", len(docs[0].elements[charlist].children), docs[0].element_count)
|
||||
|
||||
crc32 := doc_hash(docs[0])
|
||||
printf("[%v] CRC32: 0x%08x\n", "🎉" if crc32 == 0xcaa042b9 else "🤬", crc32)
|
||||
|
||||
for round in 0..<N {
|
||||
defer xml.destroy(docs[round])
|
||||
}
|
||||
}
|
||||
|
||||
doc_hash :: proc(doc: ^xml.Document, print := false) -> (crc32: u32) {
|
||||
buf: strings.Builder
|
||||
defer strings.destroy_builder(&buf)
|
||||
w := strings.to_writer(&buf)
|
||||
|
||||
xml.print(w, doc)
|
||||
tree := strings.to_string(buf)
|
||||
if print { fmt.println(tree) }
|
||||
return hash.crc32(transmute([]u8)tree)
|
||||
}
|
||||
|
||||
main :: proc() {
|
||||
using fmt
|
||||
|
||||
track: mem.Tracking_Allocator
|
||||
mem.tracking_allocator_init(&track, context.allocator)
|
||||
context.allocator = mem.tracking_allocator(&track)
|
||||
|
||||
example()
|
||||
|
||||
if len(track.allocation_map) > 0 {
|
||||
println()
|
||||
for _, v in track.allocation_map {
|
||||
printf("%v Leaked %v bytes.\n", v.location, v.size)
|
||||
}
|
||||
}
|
||||
println("Done and cleaned up!")
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
An XML 1.0 / 1.1 parser
|
||||
|
||||
Copyright 2021-2022 Jeroen van Rijn <nom@duclavier.com>.
|
||||
Made available under Odin's BSD-3 license.
|
||||
|
||||
This file contains helper functions.
|
||||
*/
|
||||
package xml
|
||||
|
||||
// Find parent's nth child with a given ident.
|
||||
find_child_by_ident :: proc(doc: ^Document, parent_id: Element_ID, ident: string, nth := 0) -> (res: Element_ID, found: bool) {
|
||||
tag := doc.elements[parent_id]
|
||||
|
||||
count := 0
|
||||
for child_id in tag.children {
|
||||
child := doc.elements[child_id]
|
||||
/*
|
||||
Skip commments. They have no name.
|
||||
*/
|
||||
if child.kind != .Element { continue }
|
||||
|
||||
/*
|
||||
If the ident matches and it's the nth such child, return it.
|
||||
*/
|
||||
if child.ident == ident {
|
||||
if count == nth { return child_id, true }
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// Find an attribute by key.
|
||||
find_attribute_val_by_key :: proc(doc: ^Document, parent_id: Element_ID, key: string) -> (val: string, found: bool) {
|
||||
tag := doc.elements[parent_id]
|
||||
|
||||
for attr in tag.attribs {
|
||||
/*
|
||||
If the ident matches, we're done. There can only ever be one attribute with the same name.
|
||||
*/
|
||||
if attr.key == key { return attr.val, true }
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,436 @@
|
||||
/*
|
||||
An XML 1.0 / 1.1 parser
|
||||
|
||||
Copyright 2021-2022 Jeroen van Rijn <nom@duclavier.com>.
|
||||
Made available under Odin's BSD-3 license.
|
||||
|
||||
A from-scratch XML implementation, loosely modeled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816).
|
||||
|
||||
List of contributors:
|
||||
Jeroen van Rijn: Initial implementation.
|
||||
*/
|
||||
package xml
|
||||
|
||||
import "core:fmt"
|
||||
import "core:unicode"
|
||||
import "core:unicode/utf8"
|
||||
|
||||
Error_Handler :: #type proc(pos: Pos, fmt: string, args: ..any)
|
||||
|
||||
Token :: struct {
|
||||
kind: Token_Kind,
|
||||
text: string,
|
||||
pos: Pos,
|
||||
}
|
||||
|
||||
Pos :: struct {
|
||||
file: string,
|
||||
offset: int, // starting at 0
|
||||
line: int, // starting at 1
|
||||
column: int, // starting at 1
|
||||
}
|
||||
|
||||
Token_Kind :: enum {
|
||||
Invalid,
|
||||
|
||||
Ident,
|
||||
Literal,
|
||||
Rune,
|
||||
String,
|
||||
|
||||
Double_Quote, // "
|
||||
Single_Quote, // '
|
||||
Colon, // :
|
||||
|
||||
Eq, // =
|
||||
Lt, // <
|
||||
Gt, // >
|
||||
Exclaim, // !
|
||||
Question, // ?
|
||||
Hash, // #
|
||||
Slash, // /
|
||||
Dash, // -
|
||||
|
||||
Open_Bracket, // [
|
||||
Close_Bracket, // ]
|
||||
|
||||
EOF,
|
||||
}
|
||||
|
||||
CDATA_START :: "<![CDATA["
|
||||
CDATA_END :: "]]>"
|
||||
|
||||
COMMENT_START :: "<!--"
|
||||
COMMENT_END :: "-->"
|
||||
|
||||
Tokenizer :: struct {
|
||||
// Immutable data
|
||||
path: string,
|
||||
src: string,
|
||||
err: Error_Handler,
|
||||
|
||||
// Tokenizing state
|
||||
ch: rune,
|
||||
offset: int,
|
||||
read_offset: int,
|
||||
line_offset: int,
|
||||
line_count: int,
|
||||
|
||||
// Mutable data
|
||||
error_count: int,
|
||||
}
|
||||
|
||||
init :: proc(t: ^Tokenizer, src: string, path: string, err: Error_Handler = default_error_handler) {
|
||||
t.src = src
|
||||
t.err = err
|
||||
t.ch = ' '
|
||||
t.offset = 0
|
||||
t.read_offset = 0
|
||||
t.line_offset = 0
|
||||
t.line_count = len(src) > 0 ? 1 : 0
|
||||
t.error_count = 0
|
||||
t.path = path
|
||||
|
||||
advance_rune(t)
|
||||
if t.ch == utf8.RUNE_BOM {
|
||||
advance_rune(t)
|
||||
}
|
||||
}
|
||||
|
||||
@(private)
|
||||
offset_to_pos :: proc(t: ^Tokenizer, offset: int) -> Pos {
|
||||
line := t.line_count
|
||||
column := offset - t.line_offset + 1
|
||||
|
||||
return Pos {
|
||||
file = t.path,
|
||||
offset = offset,
|
||||
line = line,
|
||||
column = column,
|
||||
}
|
||||
}
|
||||
|
||||
default_error_handler :: proc(pos: Pos, msg: string, args: ..any) {
|
||||
fmt.eprintf("%s(%d:%d) ", pos.file, pos.line, pos.column)
|
||||
fmt.eprintf(msg, ..args)
|
||||
fmt.eprintf("\n")
|
||||
}
|
||||
|
||||
error :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) {
|
||||
pos := offset_to_pos(t, offset)
|
||||
if t.err != nil {
|
||||
t.err(pos, msg, ..args)
|
||||
}
|
||||
t.error_count += 1
|
||||
}
|
||||
|
||||
@(optimization_mode="speed")
|
||||
advance_rune :: proc(using t: ^Tokenizer) {
|
||||
#no_bounds_check {
|
||||
/*
|
||||
Already bounds-checked here.
|
||||
*/
|
||||
if read_offset < len(src) {
|
||||
offset = read_offset
|
||||
if ch == '\n' {
|
||||
line_offset = offset
|
||||
line_count += 1
|
||||
}
|
||||
r, w := rune(src[read_offset]), 1
|
||||
switch {
|
||||
case r == 0:
|
||||
error(t, t.offset, "illegal character NUL")
|
||||
case r >= utf8.RUNE_SELF:
|
||||
r, w = #force_inline utf8.decode_rune_in_string(src[read_offset:])
|
||||
if r == utf8.RUNE_ERROR && w == 1 {
|
||||
error(t, t.offset, "illegal UTF-8 encoding")
|
||||
} else if r == utf8.RUNE_BOM && offset > 0 {
|
||||
error(t, t.offset, "illegal byte order mark")
|
||||
}
|
||||
}
|
||||
read_offset += w
|
||||
ch = r
|
||||
} else {
|
||||
offset = len(src)
|
||||
if ch == '\n' {
|
||||
line_offset = offset
|
||||
line_count += 1
|
||||
}
|
||||
ch = -1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
peek_byte :: proc(t: ^Tokenizer, offset := 0) -> byte {
|
||||
if t.read_offset+offset < len(t.src) {
|
||||
#no_bounds_check return t.src[t.read_offset+offset]
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
@(optimization_mode="speed")
|
||||
skip_whitespace :: proc(t: ^Tokenizer) {
|
||||
for {
|
||||
switch t.ch {
|
||||
case ' ', '\t', '\r', '\n':
|
||||
advance_rune(t)
|
||||
case:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@(optimization_mode="speed")
|
||||
is_letter :: proc(r: rune) -> bool {
|
||||
if r < utf8.RUNE_SELF {
|
||||
switch r {
|
||||
case '_':
|
||||
return true
|
||||
case 'A'..='Z', 'a'..='z':
|
||||
return true
|
||||
}
|
||||
}
|
||||
return unicode.is_letter(r)
|
||||
}
|
||||
|
||||
is_valid_identifier_rune :: proc(r: rune) -> bool {
|
||||
if r < utf8.RUNE_SELF {
|
||||
switch r {
|
||||
case '_', '-', ':': return true
|
||||
case 'A'..='Z', 'a'..='z': return true
|
||||
case '0'..'9': return true
|
||||
case -1: return false
|
||||
}
|
||||
}
|
||||
|
||||
if unicode.is_letter(r) || unicode.is_digit(r) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
scan_identifier :: proc(t: ^Tokenizer) -> string {
|
||||
offset := t.offset
|
||||
namespaced := false
|
||||
|
||||
for is_valid_identifier_rune(t.ch) {
|
||||
advance_rune(t)
|
||||
if t.ch == ':' {
|
||||
/*
|
||||
A namespaced attr can have at most two parts, `namespace:ident`.
|
||||
*/
|
||||
if namespaced {
|
||||
break
|
||||
}
|
||||
namespaced = true
|
||||
}
|
||||
}
|
||||
return string(t.src[offset : t.offset])
|
||||
}
|
||||
|
||||
/*
|
||||
A comment ends when we see -->, preceded by a character that's not a dash.
|
||||
"For compatibility, the string "--" (double-hyphen) must not occur within comments."
|
||||
|
||||
See: https://www.w3.org/TR/2006/REC-xml11-20060816/#dt-comment
|
||||
|
||||
Thanks to the length (4) of the comment start, we also have enough lookback,
|
||||
and the peek at the next byte asserts that there's at least one more character
|
||||
that's a `>`.
|
||||
*/
|
||||
scan_comment :: proc(t: ^Tokenizer) -> (comment: string, err: Error) {
|
||||
offset := t.offset
|
||||
|
||||
for {
|
||||
advance_rune(t)
|
||||
ch := t.ch
|
||||
|
||||
if ch < 0 {
|
||||
error(t, offset, "[parse] Comment was not terminated\n")
|
||||
return "", .Unclosed_Comment
|
||||
}
|
||||
|
||||
if string(t.src[t.offset - 1:][:2]) == "--" {
|
||||
if peek_byte(t) == '>' {
|
||||
break
|
||||
} else {
|
||||
error(t, t.offset - 1, "Invalid -- sequence in comment.\n")
|
||||
return "", .Invalid_Sequence_In_Comment
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(t, .Dash)
|
||||
expect(t, .Gt)
|
||||
|
||||
return string(t.src[offset : t.offset - 1]), .None
|
||||
}
|
||||
|
||||
/*
|
||||
Skip CDATA
|
||||
*/
|
||||
skip_cdata :: proc(t: ^Tokenizer) -> (err: Error) {
|
||||
if t.read_offset + len(CDATA_START) >= len(t.src) {
|
||||
/*
|
||||
Can't be the start of a CDATA tag.
|
||||
*/
|
||||
return .None
|
||||
}
|
||||
|
||||
if string(t.src[t.offset:][:len(CDATA_START)]) == CDATA_START {
|
||||
t.read_offset += len(CDATA_START)
|
||||
offset := t.offset
|
||||
|
||||
cdata_scan: for {
|
||||
advance_rune(t)
|
||||
if t.ch < 0 {
|
||||
error(t, offset, "[scan_string] CDATA was not terminated\n")
|
||||
return .Premature_EOF
|
||||
}
|
||||
|
||||
/*
|
||||
Scan until the end of a CDATA tag.
|
||||
*/
|
||||
if t.read_offset + len(CDATA_END) < len(t.src) {
|
||||
if string(t.src[t.offset:][:len(CDATA_END)]) == CDATA_END {
|
||||
t.read_offset += len(CDATA_END)
|
||||
break cdata_scan
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@(optimization_mode="speed")
|
||||
scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close := false, multiline := true) -> (value: string, err: Error) {
|
||||
err = .None
|
||||
|
||||
loop: for {
|
||||
ch := t.ch
|
||||
|
||||
switch ch {
|
||||
case -1:
|
||||
error(t, t.offset, "[scan_string] Premature end of file.\n")
|
||||
return "", .Premature_EOF
|
||||
|
||||
case '<':
|
||||
if peek_byte(t) == '!' {
|
||||
if peek_byte(t, 1) == '[' {
|
||||
/*
|
||||
Might be the start of a CDATA tag.
|
||||
*/
|
||||
skip_cdata(t) or_return
|
||||
} else if peek_byte(t, 1) == '-' && peek_byte(t, 2) == '-' {
|
||||
/*
|
||||
Comment start. Eat comment.
|
||||
*/
|
||||
t.read_offset += 3
|
||||
_ = scan_comment(t) or_return
|
||||
}
|
||||
}
|
||||
|
||||
case '\n':
|
||||
if !multiline {
|
||||
error(t, offset, string(t.src[offset : t.offset]))
|
||||
error(t, offset, "[scan_string] Not terminated\n")
|
||||
err = .Invalid_Tag_Value
|
||||
break loop
|
||||
}
|
||||
}
|
||||
|
||||
if t.ch == close {
|
||||
/*
|
||||
If it's not a CDATA or comment, it's the end of this body.
|
||||
*/
|
||||
break loop
|
||||
}
|
||||
advance_rune(t)
|
||||
}
|
||||
|
||||
/*
|
||||
Strip trailing whitespace.
|
||||
*/
|
||||
lit := string(t.src[offset : t.offset])
|
||||
|
||||
end := len(lit)
|
||||
eat: for ; end > 0; end -= 1 {
|
||||
ch := lit[end - 1]
|
||||
switch ch {
|
||||
case ' ', '\t', '\r', '\n':
|
||||
case:
|
||||
break eat
|
||||
}
|
||||
}
|
||||
lit = lit[:end]
|
||||
|
||||
if consume_close {
|
||||
advance_rune(t)
|
||||
}
|
||||
|
||||
/*
|
||||
TODO: Handle decoding escape characters and unboxing CDATA.
|
||||
*/
|
||||
|
||||
return lit, err
|
||||
}
|
||||
|
||||
peek :: proc(t: ^Tokenizer) -> (token: Token) {
|
||||
old := t^
|
||||
token = scan(t)
|
||||
t^ = old
|
||||
return token
|
||||
}
|
||||
|
||||
scan :: proc(t: ^Tokenizer) -> Token {
|
||||
skip_whitespace(t)
|
||||
|
||||
offset := t.offset
|
||||
|
||||
kind: Token_Kind
|
||||
err: Error
|
||||
lit: string
|
||||
pos := offset_to_pos(t, offset)
|
||||
|
||||
switch ch := t.ch; true {
|
||||
case is_letter(ch):
|
||||
lit = scan_identifier(t)
|
||||
kind = .Ident
|
||||
|
||||
case:
|
||||
advance_rune(t)
|
||||
switch ch {
|
||||
case -1:
|
||||
kind = .EOF
|
||||
|
||||
case '<': kind = .Lt
|
||||
case '>': kind = .Gt
|
||||
case '!': kind = .Exclaim
|
||||
case '?': kind = .Question
|
||||
case '=': kind = .Eq
|
||||
case '#': kind = .Hash
|
||||
case '/': kind = .Slash
|
||||
case '-': kind = .Dash
|
||||
case ':': kind = .Colon
|
||||
|
||||
case '"', '\'':
|
||||
kind = .Invalid
|
||||
|
||||
lit, err = scan_string(t, t.offset, ch, true, false)
|
||||
if err == .None {
|
||||
kind = .String
|
||||
}
|
||||
|
||||
case '\n':
|
||||
lit = "\n"
|
||||
|
||||
case:
|
||||
kind = .Invalid
|
||||
}
|
||||
}
|
||||
|
||||
if kind != .String && lit == "" {
|
||||
lit = string(t.src[offset : t.offset])
|
||||
}
|
||||
return Token{kind, lit, pos}
|
||||
}
|
||||
@@ -0,0 +1,708 @@
|
||||
/*
|
||||
An XML 1.0 / 1.1 parser
|
||||
|
||||
Copyright 2021-2022 Jeroen van Rijn <nom@duclavier.com>.
|
||||
Made available under Odin's BSD-3 license.
|
||||
|
||||
A from-scratch XML implementation, loosely modelled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816).
|
||||
|
||||
Features:
|
||||
- Supports enough of the XML 1.0/1.1 spec to handle the 99.9% of XML documents in common current usage.
|
||||
- Simple to understand and use. Small.
|
||||
|
||||
Caveats:
|
||||
- We do NOT support HTML in this package, as that may or may not be valid XML.
|
||||
If it works, great. If it doesn't, that's not considered a bug.
|
||||
|
||||
- We do NOT support UTF-16. If you have a UTF-16 XML file, please convert it to UTF-8 first. Also, our condolences.
|
||||
- <[!ELEMENT and <[!ATTLIST are not supported, and will be either ignored or return an error depending on the parser options.
|
||||
|
||||
MAYBE:
|
||||
- XML writer?
|
||||
- Serialize/deserialize Odin types?
|
||||
|
||||
List of contributors:
|
||||
Jeroen van Rijn: Initial implementation.
|
||||
*/
|
||||
package xml
|
||||
// An XML 1.0 / 1.1 parser
|
||||
|
||||
import "core:bytes"
|
||||
import "core:encoding/entity"
|
||||
import "core:intrinsics"
|
||||
import "core:mem"
|
||||
import "core:os"
|
||||
import "core:strings"
|
||||
|
||||
likely :: intrinsics.expect
|
||||
|
||||
DEFAULT_Options :: Options{
|
||||
flags = {
|
||||
.Ignore_Unsupported,
|
||||
},
|
||||
expected_doctype = "",
|
||||
}
|
||||
|
||||
Option_Flag :: enum {
|
||||
/*
|
||||
If the caller says that input may be modified, we can perform in-situ parsing.
|
||||
If this flag isn't provided, the XML parser first duplicates the input so that it can.
|
||||
*/
|
||||
Input_May_Be_Modified,
|
||||
|
||||
/*
|
||||
Document MUST start with `<?xml` prolog.
|
||||
*/
|
||||
Must_Have_Prolog,
|
||||
|
||||
/*
|
||||
Document MUST have a `<!DOCTYPE`.
|
||||
*/
|
||||
Must_Have_DocType,
|
||||
|
||||
/*
|
||||
By default we skip comments. Use this option to intern a comment on a parented Element.
|
||||
*/
|
||||
Intern_Comments,
|
||||
|
||||
/*
|
||||
How to handle unsupported parts of the specification, like <! other than <!DOCTYPE and <![CDATA[
|
||||
*/
|
||||
Error_on_Unsupported,
|
||||
Ignore_Unsupported,
|
||||
|
||||
/*
|
||||
By default CDATA tags are passed-through as-is.
|
||||
This option unwraps them when encountered.
|
||||
*/
|
||||
Unbox_CDATA,
|
||||
|
||||
/*
|
||||
By default SGML entities like `>`, ` ` and ` ` are passed-through as-is.
|
||||
This option decodes them when encountered.
|
||||
*/
|
||||
Decode_SGML_Entities,
|
||||
|
||||
/*
|
||||
If a tag body has a comment, it will be stripped unless this option is given.
|
||||
*/
|
||||
Keep_Tag_Body_Comments,
|
||||
}
|
||||
Option_Flags :: bit_set[Option_Flag; u16]
|
||||
|
||||
Document :: struct {
|
||||
elements: [dynamic]Element,
|
||||
element_count: Element_ID,
|
||||
|
||||
prolog: Attributes,
|
||||
encoding: Encoding,
|
||||
|
||||
doctype: struct {
|
||||
/*
|
||||
We only scan the <!DOCTYPE IDENT part and skip the rest.
|
||||
*/
|
||||
ident: string,
|
||||
rest: string,
|
||||
},
|
||||
|
||||
/*
|
||||
If we encounter comments before the root node, and the option to intern comments is given, this is where they'll live.
|
||||
Otherwise they'll be in the element tree.
|
||||
*/
|
||||
comments: [dynamic]string,
|
||||
|
||||
/*
|
||||
Internal
|
||||
*/
|
||||
tokenizer: ^Tokenizer,
|
||||
allocator: mem.Allocator,
|
||||
|
||||
/*
|
||||
Input. Either the original buffer, or a copy if `.Input_May_Be_Modified` isn't specified.
|
||||
*/
|
||||
input: []u8,
|
||||
strings_to_free: [dynamic]string,
|
||||
}
|
||||
|
||||
Element :: struct {
|
||||
ident: string,
|
||||
value: string,
|
||||
attribs: Attributes,
|
||||
|
||||
kind: enum {
|
||||
Element = 0,
|
||||
Comment,
|
||||
},
|
||||
|
||||
parent: Element_ID,
|
||||
children: [dynamic]Element_ID,
|
||||
}
|
||||
|
||||
Attr :: struct {
|
||||
key: string,
|
||||
val: string,
|
||||
}
|
||||
|
||||
Attributes :: [dynamic]Attr
|
||||
|
||||
Options :: struct {
|
||||
flags: Option_Flags,
|
||||
expected_doctype: string,
|
||||
}
|
||||
|
||||
Encoding :: enum {
|
||||
Unknown,
|
||||
|
||||
UTF_8,
|
||||
ISO_8859_1,
|
||||
|
||||
/*
|
||||
Aliases
|
||||
*/
|
||||
LATIN_1 = ISO_8859_1,
|
||||
}
|
||||
|
||||
Error :: enum {
|
||||
/*
|
||||
General return values.
|
||||
*/
|
||||
None = 0,
|
||||
General_Error,
|
||||
Unexpected_Token,
|
||||
Invalid_Token,
|
||||
|
||||
/*
|
||||
Couldn't find, open or read file.
|
||||
*/
|
||||
File_Error,
|
||||
|
||||
/*
|
||||
File too short.
|
||||
*/
|
||||
Premature_EOF,
|
||||
|
||||
/*
|
||||
XML-specific errors.
|
||||
*/
|
||||
No_Prolog,
|
||||
Invalid_Prolog,
|
||||
Too_Many_Prologs,
|
||||
|
||||
No_DocType,
|
||||
Too_Many_DocTypes,
|
||||
DocType_Must_Preceed_Elements,
|
||||
|
||||
/*
|
||||
If a DOCTYPE is present _or_ the caller
|
||||
asked for a specific DOCTYPE and the DOCTYPE
|
||||
and root tag don't match, we return `.Invalid_DocType`.
|
||||
*/
|
||||
Invalid_DocType,
|
||||
|
||||
Invalid_Tag_Value,
|
||||
Mismatched_Closing_Tag,
|
||||
|
||||
Unclosed_Comment,
|
||||
Comment_Before_Root_Element,
|
||||
Invalid_Sequence_In_Comment,
|
||||
|
||||
Unsupported_Version,
|
||||
Unsupported_Encoding,
|
||||
|
||||
/*
|
||||
<!FOO are usually skipped.
|
||||
*/
|
||||
Unhandled_Bang,
|
||||
|
||||
Duplicate_Attribute,
|
||||
Conflicting_Options,
|
||||
}
|
||||
|
||||
/*
|
||||
Implementation starts here.
|
||||
*/
|
||||
parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", error_handler := default_error_handler, allocator := context.allocator) -> (doc: ^Document, err: Error) {
|
||||
data := data
|
||||
context.allocator = allocator
|
||||
|
||||
opts := validate_options(options) or_return
|
||||
|
||||
/*
|
||||
If `.Input_May_Be_Modified` is not specified, we duplicate the input so that we can modify it in-place.
|
||||
*/
|
||||
if .Input_May_Be_Modified not_in opts.flags {
|
||||
data = bytes.clone(data)
|
||||
}
|
||||
|
||||
t := &Tokenizer{}
|
||||
init(t, string(data), path, error_handler)
|
||||
|
||||
doc = new(Document)
|
||||
doc.allocator = allocator
|
||||
doc.tokenizer = t
|
||||
doc.input = data
|
||||
|
||||
doc.elements = make([dynamic]Element, 1024, 1024, allocator)
|
||||
|
||||
// strings.intern_init(&doc.intern, allocator, allocator)
|
||||
|
||||
err = .Unexpected_Token
|
||||
element, parent: Element_ID
|
||||
|
||||
tag_is_open := false
|
||||
first_element := true
|
||||
open: Token
|
||||
|
||||
/*
|
||||
If a DOCTYPE is present, the root tag has to match.
|
||||
If an expected DOCTYPE is given in options (i.e. it's non-empty), the DOCTYPE (if present) and root tag have to match.
|
||||
*/
|
||||
expected_doctype := options.expected_doctype
|
||||
|
||||
loop: for {
|
||||
skip_whitespace(t)
|
||||
// NOTE(Jeroen): This is faster as a switch.
|
||||
switch t.ch {
|
||||
case '<':
|
||||
/*
|
||||
Consume peeked `<`
|
||||
*/
|
||||
advance_rune(t)
|
||||
|
||||
open = scan(t)
|
||||
// NOTE(Jeroen): We're not using a switch because this if-else chain ordered by likelihood is 2.5% faster at -o:size and -o:speed.
|
||||
if likely(open.kind, Token_Kind.Ident) == .Ident {
|
||||
/*
|
||||
e.g. <odin - Start of new element.
|
||||
*/
|
||||
element = new_element(doc)
|
||||
tag_is_open = true
|
||||
|
||||
if first_element {
|
||||
/*
|
||||
First element.
|
||||
*/
|
||||
parent = element
|
||||
first_element = false
|
||||
} else {
|
||||
append(&doc.elements[parent].children, element)
|
||||
}
|
||||
|
||||
doc.elements[element].parent = parent
|
||||
doc.elements[element].ident = open.text
|
||||
|
||||
parse_attributes(doc, &doc.elements[element].attribs) or_return
|
||||
|
||||
/*
|
||||
If a DOCTYPE is present _or_ the caller
|
||||
asked for a specific DOCTYPE and the DOCTYPE
|
||||
and root tag don't match, we return .Invalid_Root_Tag.
|
||||
*/
|
||||
if element == 0 { // Root tag?
|
||||
if len(expected_doctype) > 0 && expected_doctype != open.text {
|
||||
error(t, t.offset, "Root Tag doesn't match DOCTYPE. Expected: %v, got: %v\n", expected_doctype, open.text)
|
||||
return doc, .Invalid_DocType
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
One of these should follow:
|
||||
- `>`, which means we've just opened this tag and expect a later element to close it.
|
||||
- `/>`, which means this is an 'empty' or self-closing tag.
|
||||
*/
|
||||
end_token := scan(t)
|
||||
#partial switch end_token.kind {
|
||||
case .Gt:
|
||||
/*
|
||||
We're now the new parent.
|
||||
*/
|
||||
parent = element
|
||||
|
||||
case .Slash:
|
||||
/*
|
||||
Empty tag. Close it.
|
||||
*/
|
||||
expect(t, .Gt) or_return
|
||||
parent = doc.elements[element].parent
|
||||
element = parent
|
||||
tag_is_open = false
|
||||
|
||||
case:
|
||||
error(t, t.offset, "Expected close tag, got: %#v\n", end_token)
|
||||
return
|
||||
}
|
||||
|
||||
} else if open.kind == .Slash {
|
||||
/*
|
||||
Close tag.
|
||||
*/
|
||||
ident := expect(t, .Ident) or_return
|
||||
_ = expect(t, .Gt) or_return
|
||||
|
||||
if doc.elements[element].ident != ident.text {
|
||||
error(t, t.offset, "Mismatched Closing Tag. Expected %v, got %v\n", doc.elements[element].ident, ident.text)
|
||||
return doc, .Mismatched_Closing_Tag
|
||||
}
|
||||
parent = doc.elements[element].parent
|
||||
element = parent
|
||||
tag_is_open = false
|
||||
|
||||
} else if open.kind == .Exclaim {
|
||||
/*
|
||||
<!
|
||||
*/
|
||||
next := scan(t)
|
||||
#partial switch next.kind {
|
||||
case .Ident:
|
||||
switch next.text {
|
||||
case "DOCTYPE":
|
||||
if len(doc.doctype.ident) > 0 {
|
||||
return doc, .Too_Many_DocTypes
|
||||
}
|
||||
if doc.element_count > 0 {
|
||||
return doc, .DocType_Must_Preceed_Elements
|
||||
}
|
||||
parse_doctype(doc) or_return
|
||||
|
||||
if len(expected_doctype) > 0 && expected_doctype != doc.doctype.ident {
|
||||
error(t, t.offset, "Invalid DOCTYPE. Expected: %v, got: %v\n", expected_doctype, doc.doctype.ident)
|
||||
return doc, .Invalid_DocType
|
||||
}
|
||||
expected_doctype = doc.doctype.ident
|
||||
|
||||
case:
|
||||
if .Error_on_Unsupported in opts.flags {
|
||||
error(t, t.offset, "Unhandled: <!%v\n", next.text)
|
||||
return doc, .Unhandled_Bang
|
||||
}
|
||||
skip_element(t) or_return
|
||||
}
|
||||
|
||||
case .Dash:
|
||||
/*
|
||||
Comment: <!-- -->.
|
||||
The grammar does not allow a comment to end in --->
|
||||
*/
|
||||
expect(t, .Dash)
|
||||
comment := scan_comment(t) or_return
|
||||
|
||||
if .Intern_Comments in opts.flags {
|
||||
if len(doc.elements) == 0 {
|
||||
append(&doc.comments, comment)
|
||||
} else {
|
||||
el := new_element(doc)
|
||||
doc.elements[el].parent = element
|
||||
doc.elements[el].kind = .Comment
|
||||
doc.elements[el].value = comment
|
||||
append(&doc.elements[element].children, el)
|
||||
}
|
||||
}
|
||||
|
||||
case:
|
||||
error(t, t.offset, "Invalid Token after <!. Expected .Ident, got %#v\n", next)
|
||||
return
|
||||
}
|
||||
|
||||
} else if open.kind == .Question {
|
||||
/*
|
||||
<?xml
|
||||
*/
|
||||
next := scan(t)
|
||||
#partial switch next.kind {
|
||||
case .Ident:
|
||||
if len(next.text) == 3 && strings.to_lower(next.text, context.temp_allocator) == "xml" {
|
||||
parse_prolog(doc) or_return
|
||||
} else if len(doc.prolog) > 0 {
|
||||
/*
|
||||
We've already seen a prolog.
|
||||
*/
|
||||
return doc, .Too_Many_Prologs
|
||||
} else {
|
||||
/*
|
||||
Could be `<?xml-stylesheet`, etc. Ignore it.
|
||||
*/
|
||||
skip_element(t) or_return
|
||||
}
|
||||
case:
|
||||
error(t, t.offset, "Expected \"<?xml\", got \"<?%v\".", next.text)
|
||||
return
|
||||
}
|
||||
|
||||
} else {
|
||||
error(t, t.offset, "Invalid Token after <: %#v\n", open)
|
||||
return
|
||||
}
|
||||
|
||||
case -1:
|
||||
/*
|
||||
End of file.
|
||||
*/
|
||||
if tag_is_open {
|
||||
return doc, .Premature_EOF
|
||||
}
|
||||
break loop
|
||||
|
||||
case:
|
||||
/*
|
||||
This should be a tag's body text.
|
||||
*/
|
||||
body_text := scan_string(t, t.offset) or_return
|
||||
needs_processing := .Unbox_CDATA in opts.flags
|
||||
needs_processing |= .Decode_SGML_Entities in opts.flags
|
||||
|
||||
if !needs_processing {
|
||||
doc.elements[element].value = body_text
|
||||
continue
|
||||
}
|
||||
|
||||
decode_opts := entity.XML_Decode_Options{}
|
||||
if .Keep_Tag_Body_Comments not_in opts.flags {
|
||||
decode_opts += { .Comment_Strip }
|
||||
}
|
||||
|
||||
if .Decode_SGML_Entities not_in opts.flags {
|
||||
decode_opts += { .No_Entity_Decode }
|
||||
}
|
||||
|
||||
if .Unbox_CDATA in opts.flags {
|
||||
decode_opts += { .Unbox_CDATA }
|
||||
if .Decode_SGML_Entities in opts.flags {
|
||||
decode_opts += { .Decode_CDATA }
|
||||
}
|
||||
}
|
||||
|
||||
decoded, decode_err := entity.decode_xml(body_text, decode_opts)
|
||||
if decode_err == .None {
|
||||
doc.elements[element].value = decoded
|
||||
append(&doc.strings_to_free, decoded)
|
||||
} else {
|
||||
doc.elements[element].value = body_text
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if .Must_Have_Prolog in opts.flags && len(doc.prolog) == 0 {
|
||||
return doc, .No_Prolog
|
||||
}
|
||||
|
||||
if .Must_Have_DocType in opts.flags && len(doc.doctype.ident) == 0 {
|
||||
return doc, .No_DocType
|
||||
}
|
||||
|
||||
resize(&doc.elements, int(doc.element_count))
|
||||
return doc, .None
|
||||
}
|
||||
|
||||
parse_from_file :: proc(filename: string, options := DEFAULT_Options, error_handler := default_error_handler, allocator := context.allocator) -> (doc: ^Document, err: Error) {
|
||||
context.allocator = allocator
|
||||
options := options
|
||||
|
||||
data, data_ok := os.read_entire_file(filename)
|
||||
if !data_ok { return {}, .File_Error }
|
||||
|
||||
options.flags += { .Input_May_Be_Modified }
|
||||
|
||||
return parse_from_slice(data, options, filename, error_handler, allocator)
|
||||
}
|
||||
|
||||
parse :: proc { parse_from_file, parse_from_slice }
|
||||
|
||||
destroy :: proc(doc: ^Document) {
|
||||
if doc == nil { return }
|
||||
|
||||
for el in doc.elements {
|
||||
delete(el.attribs)
|
||||
delete(el.children)
|
||||
}
|
||||
delete(doc.elements)
|
||||
|
||||
delete(doc.prolog)
|
||||
delete(doc.comments)
|
||||
delete(doc.input)
|
||||
|
||||
for s in doc.strings_to_free {
|
||||
delete(s)
|
||||
}
|
||||
delete(doc.strings_to_free)
|
||||
|
||||
free(doc)
|
||||
}
|
||||
|
||||
/*
|
||||
Helpers.
|
||||
*/
|
||||
|
||||
validate_options :: proc(options: Options) -> (validated: Options, err: Error) {
|
||||
validated = options
|
||||
|
||||
if .Error_on_Unsupported in validated.flags && .Ignore_Unsupported in validated.flags {
|
||||
return options, .Conflicting_Options
|
||||
}
|
||||
return validated, .None
|
||||
}
|
||||
|
||||
expect :: proc(t: ^Tokenizer, kind: Token_Kind) -> (tok: Token, err: Error) {
|
||||
tok = scan(t)
|
||||
if tok.kind == kind { return tok, .None }
|
||||
|
||||
error(t, t.offset, "Expected \"%v\", got \"%v\".", kind, tok.kind)
|
||||
return tok, .Unexpected_Token
|
||||
}
|
||||
|
||||
parse_attribute :: proc(doc: ^Document) -> (attr: Attr, offset: int, err: Error) {
|
||||
assert(doc != nil)
|
||||
context.allocator = doc.allocator
|
||||
t := doc.tokenizer
|
||||
|
||||
key := expect(t, .Ident) or_return
|
||||
offset = t.offset - len(key.text)
|
||||
|
||||
_ = expect(t, .Eq) or_return
|
||||
value := expect(t, .String) or_return
|
||||
|
||||
attr.key = key.text
|
||||
attr.val = value.text
|
||||
|
||||
err = .None
|
||||
return
|
||||
}
|
||||
|
||||
check_duplicate_attributes :: proc(t: ^Tokenizer, attribs: Attributes, attr: Attr, offset: int) -> (err: Error) {
|
||||
for a in attribs {
|
||||
if attr.key == a.key {
|
||||
error(t, offset, "Duplicate attribute: %v\n", attr.key)
|
||||
return .Duplicate_Attribute
|
||||
}
|
||||
}
|
||||
return .None
|
||||
}
|
||||
|
||||
parse_attributes :: proc(doc: ^Document, attribs: ^Attributes) -> (err: Error) {
|
||||
assert(doc != nil)
|
||||
context.allocator = doc.allocator
|
||||
t := doc.tokenizer
|
||||
|
||||
for peek(t).kind == .Ident {
|
||||
attr, offset := parse_attribute(doc) or_return
|
||||
check_duplicate_attributes(t, attribs^, attr, offset) or_return
|
||||
append(attribs, attr)
|
||||
}
|
||||
skip_whitespace(t)
|
||||
return .None
|
||||
}
|
||||
|
||||
parse_prolog :: proc(doc: ^Document) -> (err: Error) {
|
||||
assert(doc != nil)
|
||||
context.allocator = doc.allocator
|
||||
t := doc.tokenizer
|
||||
|
||||
offset := t.offset
|
||||
parse_attributes(doc, &doc.prolog) or_return
|
||||
|
||||
for attr in doc.prolog {
|
||||
switch attr.key {
|
||||
case "version":
|
||||
switch attr.val {
|
||||
case "1.0", "1.1":
|
||||
case:
|
||||
error(t, offset, "[parse_prolog] Warning: Unhandled XML version: %v\n", attr.val)
|
||||
}
|
||||
|
||||
case "encoding":
|
||||
switch strings.to_lower(attr.val, context.temp_allocator) {
|
||||
case "utf-8", "utf8":
|
||||
doc.encoding = .UTF_8
|
||||
|
||||
case "latin-1", "latin1", "iso-8859-1":
|
||||
doc.encoding = .LATIN_1
|
||||
|
||||
case:
|
||||
/*
|
||||
Unrecognized encoding, assume UTF-8.
|
||||
*/
|
||||
error(t, offset, "[parse_prolog] Warning: Unrecognized encoding: %v\n", attr.val)
|
||||
}
|
||||
|
||||
case:
|
||||
// Ignored.
|
||||
}
|
||||
}
|
||||
|
||||
_ = expect(t, .Question) or_return
|
||||
_ = expect(t, .Gt) or_return
|
||||
|
||||
return .None
|
||||
}
|
||||
|
||||
skip_element :: proc(t: ^Tokenizer) -> (err: Error) {
|
||||
close := 1
|
||||
|
||||
loop: for {
|
||||
tok := scan(t)
|
||||
#partial switch tok.kind {
|
||||
case .EOF:
|
||||
error(t, t.offset, "[skip_element] Premature EOF\n")
|
||||
return .Premature_EOF
|
||||
|
||||
case .Lt:
|
||||
close += 1
|
||||
|
||||
case .Gt:
|
||||
close -= 1
|
||||
if close == 0 {
|
||||
break loop
|
||||
}
|
||||
|
||||
case:
|
||||
|
||||
}
|
||||
}
|
||||
return .None
|
||||
}
|
||||
|
||||
parse_doctype :: proc(doc: ^Document) -> (err: Error) {
|
||||
/*
|
||||
<!DOCTYPE greeting SYSTEM "hello.dtd">
|
||||
|
||||
<!DOCTYPE greeting [
|
||||
<!ELEMENT greeting (#PCDATA)>
|
||||
]>
|
||||
*/
|
||||
assert(doc != nil)
|
||||
context.allocator = doc.allocator
|
||||
t := doc.tokenizer
|
||||
|
||||
tok := expect(t, .Ident) or_return
|
||||
doc.doctype.ident = tok.text
|
||||
|
||||
skip_whitespace(t)
|
||||
offset := t.offset
|
||||
skip_element(t) or_return
|
||||
|
||||
/*
|
||||
-1 because the current offset is that of the closing tag, so the rest of the DOCTYPE tag ends just before it.
|
||||
*/
|
||||
doc.doctype.rest = string(t.src[offset : t.offset - 1])
|
||||
return .None
|
||||
}
|
||||
|
||||
Element_ID :: u32
|
||||
|
||||
new_element :: proc(doc: ^Document) -> (id: Element_ID) {
|
||||
element_space := len(doc.elements)
|
||||
|
||||
// Need to resize
|
||||
if int(doc.element_count) + 1 > element_space {
|
||||
if element_space < 65536 {
|
||||
element_space *= 2
|
||||
} else {
|
||||
element_space += 65536
|
||||
}
|
||||
resize(&doc.elements, element_space)
|
||||
}
|
||||
|
||||
cur := doc.element_count
|
||||
doc.element_count += 1
|
||||
|
||||
return cur
|
||||
}
|
||||
@@ -365,6 +365,20 @@ QOI_Info :: struct {
|
||||
header: QOI_Header,
|
||||
}
|
||||
|
||||
TGA_Header :: struct #packed {
|
||||
id_length: u8,
|
||||
color_map_type: u8,
|
||||
data_type_code: u8,
|
||||
color_map_origin: u16le,
|
||||
color_map_length: u16le,
|
||||
color_map_depth: u8,
|
||||
origin: [2]u16le,
|
||||
dimensions: [2]u16le,
|
||||
bits_per_pixel: u8,
|
||||
image_descriptor: u8,
|
||||
}
|
||||
#assert(size_of(TGA_Header) == 18)
|
||||
|
||||
// Function to help with image buffer calculations
|
||||
compute_buffer_size :: proc(width, height, channels, depth: int, extra_row_bytes := int(0)) -> (size: int) {
|
||||
size = ((((channels * width * depth) + 7) >> 3) + extra_row_bytes) * height
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
Copyright 2022 Jeroen van Rijn <nom@duclavier.com>.
|
||||
Made available under Odin's BSD-3 license.
|
||||
|
||||
List of contributors:
|
||||
Jeroen van Rijn: Initial implementation.
|
||||
*/
|
||||
|
||||
|
||||
// package tga implements a TGA image writer for 8-bit RGB and RGBA images.
|
||||
package tga
|
||||
|
||||
import "core:mem"
|
||||
import "core:image"
|
||||
import "core:compress"
|
||||
import "core:bytes"
|
||||
import "core:os"
|
||||
|
||||
Error :: image.Error
|
||||
General :: compress.General_Error
|
||||
Image :: image.Image
|
||||
Options :: image.Options
|
||||
|
||||
RGB_Pixel :: image.RGB_Pixel
|
||||
RGBA_Pixel :: image.RGBA_Pixel
|
||||
|
||||
save_to_memory :: proc(output: ^bytes.Buffer, img: ^Image, options := Options{}, allocator := context.allocator) -> (err: Error) {
|
||||
context.allocator = allocator
|
||||
|
||||
if img == nil {
|
||||
return .Invalid_Input_Image
|
||||
}
|
||||
|
||||
if output == nil {
|
||||
return .Invalid_Output
|
||||
}
|
||||
|
||||
pixels := img.width * img.height
|
||||
if pixels == 0 || pixels > image.MAX_DIMENSIONS || img.width > 65535 || img.height > 65535 {
|
||||
return .Invalid_Input_Image
|
||||
}
|
||||
|
||||
// Our TGA writer supports only 8-bit images with 3 or 4 channels.
|
||||
if img.depth != 8 || img.channels < 3 || img.channels > 4 {
|
||||
return .Invalid_Input_Image
|
||||
}
|
||||
|
||||
if img.channels * pixels != len(img.pixels.buf) {
|
||||
return .Invalid_Input_Image
|
||||
}
|
||||
|
||||
written := 0
|
||||
|
||||
// Calculate and allocate necessary space.
|
||||
necessary := pixels * img.channels + size_of(image.TGA_Header)
|
||||
|
||||
if !resize(&output.buf, necessary) {
|
||||
return General.Resize_Failed
|
||||
}
|
||||
|
||||
header := image.TGA_Header{
|
||||
data_type_code = 0x02, // Color, uncompressed.
|
||||
dimensions = {u16le(img.width), u16le(img.height)},
|
||||
bits_per_pixel = u8(img.depth * img.channels),
|
||||
image_descriptor = 1 << 5, // Origin is top left.
|
||||
}
|
||||
header_bytes := transmute([size_of(image.TGA_Header)]u8)header
|
||||
|
||||
copy(output.buf[written:], header_bytes[:])
|
||||
written += size_of(image.TGA_Header)
|
||||
|
||||
/*
|
||||
Encode loop starts here.
|
||||
*/
|
||||
if img.channels == 3 {
|
||||
pix := mem.slice_data_cast([]RGB_Pixel, img.pixels.buf[:])
|
||||
out := mem.slice_data_cast([]RGB_Pixel, output.buf[written:])
|
||||
for p, i in pix {
|
||||
out[i] = p.bgr
|
||||
}
|
||||
} else if img.channels == 4 {
|
||||
pix := mem.slice_data_cast([]RGBA_Pixel, img.pixels.buf[:])
|
||||
out := mem.slice_data_cast([]RGBA_Pixel, output.buf[written:])
|
||||
for p, i in pix {
|
||||
out[i] = p.bgra
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
save_to_file :: proc(output: string, img: ^Image, options := Options{}, allocator := context.allocator) -> (err: Error) {
|
||||
context.allocator = allocator
|
||||
|
||||
out := &bytes.Buffer{}
|
||||
defer bytes.buffer_destroy(out)
|
||||
|
||||
save_to_memory(out, img, options) or_return
|
||||
write_ok := os.write_entire_file(output, out.buf[:])
|
||||
|
||||
return nil if write_ok else General.Cannot_Open_File
|
||||
}
|
||||
|
||||
save :: proc{save_to_memory, save_to_file}
|
||||
@@ -41,6 +41,10 @@ mem_copy_non_overlapping :: proc(dst, src: rawptr, len: int) ---
|
||||
mem_zero :: proc(ptr: rawptr, len: int) ---
|
||||
mem_zero_volatile :: proc(ptr: rawptr, len: int) ---
|
||||
|
||||
// prefer [^]T operations if possible
|
||||
ptr_offset :: proc(ptr: ^$T, offset: int) -> ^T ---
|
||||
ptr_sub :: proc(a, b: ^$T) -> int ---
|
||||
|
||||
unaligned_load :: proc(src: ^$T) -> T ---
|
||||
unaligned_store :: proc(dst: ^$T, val: T) -> T ---
|
||||
|
||||
@@ -82,6 +86,7 @@ atomic_store_explicit :: proc(dst: ^$T, val: T, order: Atomic_Memory_Order) ---
|
||||
atomic_load :: proc(dst: ^$T) -> T ---
|
||||
atomic_load_explicit :: proc(dst: ^$T, order: Atomic_Memory_Order) -> T ---
|
||||
|
||||
// fetch then operator
|
||||
atomic_add :: proc(dst; ^$T, val: T) -> T ---
|
||||
atomic_add_explicit :: proc(dst; ^$T, val: T, order: Atomic_Memory_Order) -> T ---
|
||||
atomic_sub :: proc(dst; ^$T, val: T) -> T ---
|
||||
@@ -119,19 +124,20 @@ type_is_string :: proc($T: typeid) -> bool ---
|
||||
type_is_typeid :: proc($T: typeid) -> bool ---
|
||||
type_is_any :: proc($T: typeid) -> bool ---
|
||||
|
||||
type_is_endian_platform :: proc($T: typeid) -> bool ---
|
||||
type_is_endian_little :: proc($T: typeid) -> bool ---
|
||||
type_is_endian_big :: proc($T: typeid) -> bool ---
|
||||
type_is_unsigned :: proc($T: typeid) -> bool ---
|
||||
type_is_numeric :: proc($T: typeid) -> bool ---
|
||||
type_is_ordered :: proc($T: typeid) -> bool ---
|
||||
type_is_ordered_numeric :: proc($T: typeid) -> bool ---
|
||||
type_is_indexable :: proc($T: typeid) -> bool ---
|
||||
type_is_sliceable :: proc($T: typeid) -> bool ---
|
||||
type_is_comparable :: proc($T: typeid) -> bool ---
|
||||
type_is_simple_compare :: proc($T: typeid) -> bool --- // easily compared using memcmp (== and !=)
|
||||
type_is_dereferenceable :: proc($T: typeid) -> bool ---
|
||||
type_is_valid_map_key :: proc($T: typeid) -> bool ---
|
||||
type_is_endian_platform :: proc($T: typeid) -> bool ---
|
||||
type_is_endian_little :: proc($T: typeid) -> bool ---
|
||||
type_is_endian_big :: proc($T: typeid) -> bool ---
|
||||
type_is_unsigned :: proc($T: typeid) -> bool ---
|
||||
type_is_numeric :: proc($T: typeid) -> bool ---
|
||||
type_is_ordered :: proc($T: typeid) -> bool ---
|
||||
type_is_ordered_numeric :: proc($T: typeid) -> bool ---
|
||||
type_is_indexable :: proc($T: typeid) -> bool ---
|
||||
type_is_sliceable :: proc($T: typeid) -> bool ---
|
||||
type_is_comparable :: proc($T: typeid) -> bool ---
|
||||
type_is_simple_compare :: proc($T: typeid) -> bool --- // easily compared using memcmp (== and !=)
|
||||
type_is_dereferenceable :: proc($T: typeid) -> bool ---
|
||||
type_is_valid_map_key :: proc($T: typeid) -> bool ---
|
||||
type_is_valid_matrix_elements :: proc($T: typeid) -> bool ---
|
||||
|
||||
type_is_named :: proc($T: typeid) -> bool ---
|
||||
type_is_pointer :: proc($T: typeid) -> bool ---
|
||||
@@ -146,6 +152,7 @@ type_is_enum :: proc($T: typeid) -> bool ---
|
||||
type_is_proc :: proc($T: typeid) -> bool ---
|
||||
type_is_bit_set :: proc($T: typeid) -> bool ---
|
||||
type_is_simd_vector :: proc($T: typeid) -> bool ---
|
||||
type_is_matrix :: proc($T: typeid) -> bool ---
|
||||
|
||||
type_has_nil :: proc($T: typeid) -> bool ---
|
||||
|
||||
@@ -153,6 +160,7 @@ type_is_specialization_of :: proc($T, $S: typeid) -> bool ---
|
||||
type_is_variant_of :: proc($U, $V: typeid) -> bool where type_is_union(U) ---
|
||||
|
||||
type_has_field :: proc($T: typeid, $name: string) -> bool ---
|
||||
type_field_type :: proc($T: typeid, $name: string) -> typeid ---
|
||||
|
||||
type_proc_parameter_count :: proc($T: typeid) -> int where type_is_proc(T) ---
|
||||
type_proc_return_count :: proc($T: typeid) -> int where type_is_proc(T) ---
|
||||
@@ -160,20 +168,41 @@ type_proc_return_count :: proc($T: typeid) -> int where type_is_proc(T) ---
|
||||
type_proc_parameter_type :: proc($T: typeid, index: int) -> typeid where type_is_proc(T) ---
|
||||
type_proc_return_type :: proc($T: typeid, index: int) -> typeid where type_is_proc(T) ---
|
||||
|
||||
type_struct_field_count :: proc($T: typeid) -> int where type_is_struct(T) ---
|
||||
|
||||
type_polymorphic_record_parameter_count :: proc($T: typeid) -> typeid ---
|
||||
type_polymorphic_record_parameter_value :: proc($T: typeid, index: int) -> $V ---
|
||||
|
||||
type_is_specialized_polymorphic_record :: proc($T: typeid) -> bool ---
|
||||
type_is_unspecialized_polymorphic_record :: proc($T: typeid) -> bool ---
|
||||
|
||||
type_is_subtype_of :: proc($T, $U: typeid) -> bool ---
|
||||
|
||||
type_field_index_of :: proc($T: typeid, $name: string) -> uintptr ---
|
||||
|
||||
type_equal_proc :: proc($T: typeid) -> (equal: proc "contextless" (rawptr, rawptr) -> bool) where type_is_comparable(T) ---
|
||||
type_hasher_proc :: proc($T: typeid) -> (hasher: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr) where type_is_comparable(T) ---
|
||||
|
||||
constant_utf16_cstring :: proc($literal: string) -> [^]u16 ---
|
||||
|
||||
// WASM targets only
|
||||
wasm_memory_grow :: proc(index, delta: uintptr) -> int ---
|
||||
wasm_memory_size :: proc(index: uintptr) -> int ---
|
||||
|
||||
|
||||
// Darwin targets only
|
||||
objc_object :: struct{}
|
||||
objc_selector :: struct{}
|
||||
objc_class :: struct{}
|
||||
objc_id :: ^objc_object
|
||||
objc_SEL :: ^objc_selector
|
||||
objc_Class :: ^objc_class
|
||||
|
||||
objc_find_selector :: proc($name: string) -> objc_SEL ---
|
||||
objc_register_selector :: proc($name: string) -> objc_SEL ---
|
||||
objc_find_class :: proc($name: string) -> objc_Class ---
|
||||
objc_register_class :: proc($name: string) -> objc_Class ---
|
||||
|
||||
// Internal compiler use only
|
||||
|
||||
__entry_point :: proc() ---
|
||||
@@ -479,21 +479,21 @@ angle_from_quaternion_f16 :: proc(q: Quaternionf16) -> f16 {
|
||||
return math.asin(q.x*q.x + q.y*q.y + q.z*q.z) * 2
|
||||
}
|
||||
|
||||
return math.cos(q.x) * 2
|
||||
return math.acos(q.w) * 2
|
||||
}
|
||||
angle_from_quaternion_f32 :: proc(q: Quaternionf32) -> f32 {
|
||||
if abs(q.w) > math.SQRT_THREE*0.5 {
|
||||
return math.asin(q.x*q.x + q.y*q.y + q.z*q.z) * 2
|
||||
}
|
||||
|
||||
return math.cos(q.x) * 2
|
||||
return math.acos(q.w) * 2
|
||||
}
|
||||
angle_from_quaternion_f64 :: proc(q: Quaternionf64) -> f64 {
|
||||
if abs(q.w) > math.SQRT_THREE*0.5 {
|
||||
return math.asin(q.x*q.x + q.y*q.y + q.z*q.z) * 2
|
||||
}
|
||||
|
||||
return math.cos(q.x) * 2
|
||||
return math.acos(q.w) * 2
|
||||
}
|
||||
angle_from_quaternion :: proc{
|
||||
angle_from_quaternion_f16,
|
||||
|
||||
@@ -6,24 +6,7 @@ import "core:runtime"
|
||||
nil_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode,
|
||||
size, alignment: int,
|
||||
old_memory: rawptr, old_size: int, loc := #caller_location) -> ([]byte, Allocator_Error) {
|
||||
switch mode {
|
||||
case .Alloc:
|
||||
return nil, .Out_Of_Memory
|
||||
case .Free:
|
||||
return nil, .None
|
||||
case .Free_All:
|
||||
return nil, .Mode_Not_Implemented
|
||||
case .Resize:
|
||||
if size == 0 {
|
||||
return nil, .None
|
||||
}
|
||||
return nil, .Out_Of_Memory
|
||||
case .Query_Features:
|
||||
return nil, .Mode_Not_Implemented
|
||||
case .Query_Info:
|
||||
return nil, .Mode_Not_Implemented
|
||||
}
|
||||
return nil, .None
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
nil_allocator :: proc() -> Allocator {
|
||||
|
||||
@@ -120,7 +120,7 @@ alloc_from_memory_block :: proc(block: ^Memory_Block, min_size, alignment: int)
|
||||
do_commit_if_necessary :: proc(block: ^Memory_Block, size: uint) -> (err: Allocator_Error) {
|
||||
if block.committed - block.used < size {
|
||||
pmblock := (^Platform_Memory_Block)(block)
|
||||
base_offset := uint(uintptr(block) - uintptr(pmblock))
|
||||
base_offset := uint(uintptr(pmblock.block.base) - uintptr(pmblock))
|
||||
platform_total_commit := base_offset + block.used + size
|
||||
|
||||
assert(pmblock.committed <= pmblock.reserved)
|
||||
|
||||
@@ -4,6 +4,8 @@ package filepath
|
||||
|
||||
import "core:strings"
|
||||
|
||||
SEPARATOR_CHARS :: `/\`
|
||||
|
||||
// is_separator checks whether the byte is a valid separator character
|
||||
is_separator :: proc(c: byte) -> bool {
|
||||
switch c {
|
||||
@@ -69,6 +71,16 @@ volume_name_len :: proc(path: string) -> int {
|
||||
return 0
|
||||
}
|
||||
|
||||
/*
|
||||
Gets the file name and extension from a path.
|
||||
|
||||
i.e:
|
||||
'path/to/name.tar.gz' -> 'name.tar.gz'
|
||||
'path/to/name.txt' -> 'name.txt'
|
||||
'path/to/name' -> 'name'
|
||||
|
||||
Returns "." if the path is an empty string.
|
||||
*/
|
||||
base :: proc(path: string) -> string {
|
||||
if path == "" {
|
||||
return "."
|
||||
@@ -94,6 +106,118 @@ base :: proc(path: string) -> string {
|
||||
return path
|
||||
}
|
||||
|
||||
/*
|
||||
Gets the name of a file from a path.
|
||||
|
||||
The stem of a file is such that stem(path) + ext(path) = base(path).
|
||||
|
||||
Only the last dot is considered when splitting the file extension.
|
||||
See `short_stem`.
|
||||
|
||||
i.e:
|
||||
'name.tar.gz' -> 'name.tar'
|
||||
'name.txt' -> 'name'
|
||||
|
||||
Returns an empty string if there is no stem. e.g: '.gitignore'.
|
||||
Returns an empty string if there's a trailing path separator.
|
||||
*/
|
||||
stem :: proc(path: string) -> string {
|
||||
if len(path) > 0 && is_separator(path[len(path) - 1]) {
|
||||
// NOTE(tetra): Trailing separator
|
||||
return ""
|
||||
}
|
||||
|
||||
// NOTE(tetra): Get the basename
|
||||
path := path
|
||||
if i := strings.last_index_any(path, SEPARATOR_CHARS); i != -1 {
|
||||
path = path[i+1:]
|
||||
}
|
||||
|
||||
if i := strings.last_index_byte(path, '.'); i != -1 {
|
||||
return path[:i]
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
/*
|
||||
Gets the name of a file from a path.
|
||||
|
||||
The short stem is such that short_stem(path) + long_ext(path) = base(path).
|
||||
|
||||
The first dot is used to split off the file extension, unlike `stem` which uses the last dot.
|
||||
|
||||
i.e:
|
||||
'name.tar.gz' -> 'name'
|
||||
'name.txt' -> 'name'
|
||||
|
||||
Returns an empty string if there is no stem. e.g: '.gitignore'.
|
||||
Returns an empty string if there's a trailing path separator.
|
||||
*/
|
||||
short_stem :: proc(path: string) -> string {
|
||||
s := stem(path)
|
||||
if i := strings.index_byte(s, '.'); i != -1 {
|
||||
return s[:i]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
/*
|
||||
Gets the file extension from a path, including the dot.
|
||||
|
||||
The file extension is such that stem(path) + ext(path) = base(path).
|
||||
|
||||
Only the last dot is considered when splitting the file extension.
|
||||
See `long_ext`.
|
||||
|
||||
i.e:
|
||||
'name.tar.gz' -> '.gz'
|
||||
'name.txt' -> '.txt'
|
||||
|
||||
Returns an empty string if there is no dot.
|
||||
Returns an empty string if there is a trailing path separator.
|
||||
*/
|
||||
ext :: proc(path: string) -> string {
|
||||
for i := len(path)-1; i >= 0 && !is_separator(path[i]); i -= 1 {
|
||||
if path[i] == '.' {
|
||||
return path[i:]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
/*
|
||||
Gets the file extension from a path, including the dot.
|
||||
|
||||
The long file extension is such that short_stem(path) + long_ext(path) = base(path).
|
||||
|
||||
The first dot is used to split off the file extension, unlike `ext` which uses the last dot.
|
||||
|
||||
i.e:
|
||||
'name.tar.gz' -> '.tar.gz'
|
||||
'name.txt' -> '.txt'
|
||||
|
||||
Returns an empty string if there is no dot.
|
||||
Returns an empty string if there is a trailing path separator.
|
||||
*/
|
||||
long_ext :: proc(path: string) -> string {
|
||||
if len(path) > 0 && is_separator(path[len(path) - 1]) {
|
||||
// NOTE(tetra): Trailing separator
|
||||
return ""
|
||||
}
|
||||
|
||||
// NOTE(tetra): Get the basename
|
||||
path := path
|
||||
if i := strings.last_index_any(path, SEPARATOR_CHARS); i != -1 {
|
||||
path = path[i+1:]
|
||||
}
|
||||
|
||||
if i := strings.index_byte(path, '.'); i != -1 {
|
||||
return path[i:]
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
clean :: proc(path: string, allocator := context.allocator) -> string {
|
||||
context.allocator = allocator
|
||||
@@ -189,15 +313,6 @@ to_slash :: proc(path: string, allocator := context.allocator) -> (new_path: str
|
||||
return strings.replace_all(path, SEPARATOR_STRING, "/", allocator)
|
||||
}
|
||||
|
||||
ext :: proc(path: string) -> string {
|
||||
for i := len(path)-1; i >= 0 && !is_separator(path[i]); i -= 1 {
|
||||
if path[i] == '.' {
|
||||
return path[i:]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
|
||||
Relative_Error :: enum {
|
||||
None,
|
||||
|
||||
@@ -5,6 +5,16 @@ import "core:intrinsics"
|
||||
@builtin
|
||||
Maybe :: union($T: typeid) #maybe {T}
|
||||
|
||||
|
||||
@builtin
|
||||
container_of :: #force_inline proc "contextless" (ptr: $P/^$Field_Type, $T: typeid, $field_name: string) -> ^T
|
||||
where intrinsics.type_has_field(T, field_name),
|
||||
intrinsics.type_field_type(T, field_name) == Field_Type {
|
||||
offset :: offset_of_by_string(T, field_name)
|
||||
return (^T)(uintptr(ptr) - offset) if ptr != nil else nil
|
||||
}
|
||||
|
||||
|
||||
@thread_local global_default_temp_allocator_data: Default_Temp_Allocator
|
||||
|
||||
@builtin
|
||||
|
||||
@@ -15,7 +15,7 @@ clone :: proc(s: string, allocator := context.allocator, loc := #caller_location
|
||||
}
|
||||
|
||||
// returns a clone of the string `s` allocated using the `allocator` as a cstring
|
||||
// a nul byte is appended to the clone, to make the cstring safe
|
||||
// a nul byte is appended to the clone, to make the cstring safe
|
||||
clone_to_cstring :: proc(s: string, allocator := context.allocator, loc := #caller_location) -> cstring {
|
||||
c := make([]byte, len(s)+1, allocator, loc)
|
||||
copy(c, s)
|
||||
@@ -37,7 +37,7 @@ string_from_nul_terminated_ptr :: proc(ptr: ^byte, len: int) -> string {
|
||||
return s
|
||||
}
|
||||
|
||||
// returns the raw ^byte start of the string `str`
|
||||
// returns the raw ^byte start of the string `str`
|
||||
ptr_from_string :: proc(str: string) -> ^byte {
|
||||
d := transmute(mem.Raw_String)str
|
||||
return d.data
|
||||
@@ -969,7 +969,7 @@ count :: proc(s, substr: string) -> int {
|
||||
repeats the string `s` multiple `count` times and returns the allocated string
|
||||
panics when `count` is below 0
|
||||
|
||||
strings.repeat("abc", 2) -> "abcabc"
|
||||
strings.repeat("abc", 2) -> "abcabc"
|
||||
*/
|
||||
repeat :: proc(s: string, count: int, allocator := context.allocator) -> string {
|
||||
if count < 0 {
|
||||
@@ -1378,7 +1378,7 @@ split_multi :: proc(s: string, substrs: []string, allocator := context.allocator
|
||||
|
||||
// skip when no results
|
||||
if substrings_found < 1 {
|
||||
return
|
||||
return
|
||||
}
|
||||
|
||||
buf = make([]string, substrings_found + 1, allocator)
|
||||
@@ -1809,3 +1809,65 @@ fields_iterator :: proc(s: ^string) -> (field: string, ok: bool) {
|
||||
s^ = s[len(s):]
|
||||
return
|
||||
}
|
||||
|
||||
// `levenshtein_distance` returns the Levenshtein edit distance between 2 strings.
|
||||
// This is a single-row-version of the Wagner–Fischer algorithm, based on C code by Martin Ettl.
|
||||
// Note: allocator isn't used if the length of string b in runes is smaller than 64.
|
||||
levenshtein_distance :: proc(a, b: string, allocator := context.allocator) -> int {
|
||||
LEVENSHTEIN_DEFAULT_COSTS: []int : {
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
|
||||
10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
|
||||
20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
|
||||
30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
|
||||
40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
|
||||
50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
|
||||
60, 61, 62, 63,
|
||||
}
|
||||
|
||||
m, n := utf8.rune_count_in_string(a), utf8.rune_count_in_string(b)
|
||||
|
||||
if m == 0 {
|
||||
return n
|
||||
}
|
||||
if n == 0 {
|
||||
return m
|
||||
}
|
||||
|
||||
costs: []int
|
||||
|
||||
if n + 1 > len(LEVENSHTEIN_DEFAULT_COSTS) {
|
||||
costs = make([]int, n + 1, allocator)
|
||||
for k in 0..=n {
|
||||
costs[k] = k
|
||||
}
|
||||
} else {
|
||||
costs = LEVENSHTEIN_DEFAULT_COSTS
|
||||
}
|
||||
|
||||
defer if n + 1 > len(LEVENSHTEIN_DEFAULT_COSTS) {
|
||||
delete(costs, allocator)
|
||||
}
|
||||
|
||||
i: int
|
||||
for c1 in a {
|
||||
costs[0] = i + 1
|
||||
corner := i
|
||||
j: int
|
||||
for c2 in b {
|
||||
upper := costs[j + 1]
|
||||
if c1 == c2 {
|
||||
costs[j + 1] = corner
|
||||
} else {
|
||||
t := upper if upper < corner else corner
|
||||
costs[j + 1] = (costs[j] if costs[j] < t else t) + 1
|
||||
}
|
||||
|
||||
corner = upper
|
||||
j += 1
|
||||
}
|
||||
|
||||
i += 1
|
||||
}
|
||||
|
||||
return costs[n]
|
||||
}
|
||||
|
||||
@@ -400,30 +400,28 @@ atomic_cond_broadcast :: proc(c: ^Atomic_Cond) {
|
||||
//
|
||||
// An Atomic_Sema must not be copied after first use
|
||||
Atomic_Sema :: struct {
|
||||
mutex: Atomic_Mutex,
|
||||
cond: Atomic_Cond,
|
||||
count: int,
|
||||
count: Futex,
|
||||
}
|
||||
|
||||
atomic_sema_post :: proc(s: ^Atomic_Sema, count := 1) {
|
||||
atomic_mutex_lock(&s.mutex)
|
||||
defer atomic_mutex_unlock(&s.mutex)
|
||||
|
||||
s.count += count
|
||||
atomic_cond_signal(&s.cond)
|
||||
atomic_add_explicit(&s.count, Futex(count), .Release)
|
||||
if count == 1 {
|
||||
futex_signal(&s.count)
|
||||
} else {
|
||||
futex_broadcast(&s.count)
|
||||
}
|
||||
}
|
||||
|
||||
atomic_sema_wait :: proc(s: ^Atomic_Sema) {
|
||||
atomic_mutex_lock(&s.mutex)
|
||||
defer atomic_mutex_unlock(&s.mutex)
|
||||
|
||||
for s.count == 0 {
|
||||
atomic_cond_wait(&s.cond, &s.mutex)
|
||||
}
|
||||
|
||||
s.count -= 1
|
||||
if s.count > 0 {
|
||||
atomic_cond_signal(&s.cond)
|
||||
for {
|
||||
original_count := atomic_load_explicit(&s.count, .Relaxed)
|
||||
for original_count == 0 {
|
||||
futex_wait(&s.count, u32(original_count))
|
||||
original_count = s.count
|
||||
}
|
||||
if original_count == atomic_compare_exchange_strong_explicit(&s.count, original_count, original_count-1, .Acquire, .Acquire) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -431,25 +429,22 @@ atomic_sema_wait_with_timeout :: proc(s: ^Atomic_Sema, duration: time.Duration)
|
||||
if duration <= 0 {
|
||||
return false
|
||||
}
|
||||
atomic_mutex_lock(&s.mutex)
|
||||
defer atomic_mutex_unlock(&s.mutex)
|
||||
|
||||
start := time.tick_now()
|
||||
for {
|
||||
|
||||
for s.count == 0 {
|
||||
remaining := duration - time.tick_since(start)
|
||||
if remaining < 0 {
|
||||
return false
|
||||
original_count := atomic_load_explicit(&s.count, .Relaxed)
|
||||
for start := time.tick_now(); original_count == 0; /**/ {
|
||||
remaining := duration - time.tick_since(start)
|
||||
if remaining < 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
if !futex_wait_with_timeout(&s.count, u32(original_count), remaining) {
|
||||
return false
|
||||
}
|
||||
original_count = s.count
|
||||
}
|
||||
|
||||
if !atomic_cond_wait_with_timeout(&s.cond, &s.mutex, remaining) {
|
||||
return false
|
||||
if original_count == atomic_compare_exchange_strong_explicit(&s.count, original_count, original_count-1, .Acquire, .Acquire) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
s.count -= 1
|
||||
if s.count > 0 {
|
||||
atomic_cond_signal(&s.cond)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -6,53 +6,19 @@ import "core:time"
|
||||
|
||||
when #config(ODIN_SYNC_SEMA_USE_FUTEX, true) {
|
||||
_Sema :: struct {
|
||||
count: Futex,
|
||||
atomic: Atomic_Sema,
|
||||
}
|
||||
|
||||
_sema_post :: proc(s: ^Sema, count := 1) {
|
||||
atomic_add_explicit(&s.impl.count, Futex(count), .Release)
|
||||
if count == 1 {
|
||||
futex_signal(&s.impl.count)
|
||||
} else {
|
||||
futex_broadcast(&s.impl.count)
|
||||
}
|
||||
atomic_sema_post(&s.impl.atomic, count)
|
||||
}
|
||||
|
||||
_sema_wait :: proc(s: ^Sema) {
|
||||
for {
|
||||
original_count := atomic_load_explicit(&s.impl.count, .Relaxed)
|
||||
for original_count == 0 {
|
||||
futex_wait(&s.impl.count, u32(original_count))
|
||||
original_count = s.impl.count
|
||||
}
|
||||
if original_count == atomic_compare_exchange_strong_explicit(&s.impl.count, original_count, original_count-1, .Acquire, .Acquire) {
|
||||
return
|
||||
}
|
||||
}
|
||||
atomic_sema_wait(&s.impl.atomic)
|
||||
}
|
||||
|
||||
_sema_wait_with_timeout :: proc(s: ^Sema, duration: time.Duration) -> bool {
|
||||
if duration <= 0 {
|
||||
return false
|
||||
}
|
||||
for {
|
||||
|
||||
original_count := atomic_load_explicit(&s.impl.count, .Relaxed)
|
||||
for start := time.tick_now(); original_count == 0; /**/ {
|
||||
remaining := duration - time.tick_since(start)
|
||||
if remaining < 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
if !futex_wait_with_timeout(&s.impl.count, u32(original_count), remaining) {
|
||||
return false
|
||||
}
|
||||
original_count = s.impl.count
|
||||
}
|
||||
if original_count == atomic_compare_exchange_strong_explicit(&s.impl.count, original_count, original_count-1, .Acquire, .Acquire) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return atomic_sema_wait_with_timeout(&s.impl.atomic, duration)
|
||||
}
|
||||
} else {
|
||||
_Sema :: struct {
|
||||
|
||||
@@ -62,6 +62,13 @@ foreign kernel32 {
|
||||
GetCurrentProcessId :: proc() -> DWORD ---
|
||||
GetCurrentThread :: proc() -> HANDLE ---
|
||||
GetCurrentThreadId :: proc() -> DWORD ---
|
||||
GetProcessTimes :: proc(
|
||||
hProcess: HANDLE,
|
||||
lpCreationTime: LPFILETIME,
|
||||
lpExitTime: LPFILETIME,
|
||||
lpKernelTime: LPFILETIME,
|
||||
lpUserTime: LPFILETIME,
|
||||
) -> BOOL ---
|
||||
GetStdHandle :: proc(which: DWORD) -> HANDLE ---
|
||||
ExitProcess :: proc(uExitCode: c_uint) -> ! ---
|
||||
DeviceIoControl :: proc(
|
||||
|
||||
@@ -60,6 +60,12 @@ foreign user32 {
|
||||
DestroyWindow :: proc(hWnd: HWND) -> BOOL ---
|
||||
|
||||
ShowWindow :: proc(hWnd: HWND, nCmdShow: c_int) -> BOOL ---
|
||||
BringWindowToTop :: proc(hWnd: HWND) -> BOOL ---
|
||||
GetTopWindow :: proc(hWnd: HWND) -> HWND ---
|
||||
SetForegroundWindow :: proc(hWnd: HWND) -> BOOL ---
|
||||
GetForegroundWindow :: proc() -> HWND ---
|
||||
SetActiveWindow :: proc(hWnd: HWND) -> HWND ---
|
||||
GetActiveWindow :: proc() -> HWND ---
|
||||
|
||||
GetMessageA :: proc(lpMsg: ^MSG, hWnd: HWND, wMsgFilterMin: UINT, wMsgFilterMax: UINT) -> BOOL ---
|
||||
GetMessageW :: proc(lpMsg: ^MSG, hWnd: HWND, wMsgFilterMin: UINT, wMsgFilterMax: UINT) -> BOOL ---
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
//+ignore
|
||||
package i18n
|
||||
|
||||
/*
|
||||
The i18n package is flexible and easy to use.
|
||||
|
||||
It has one call to get a translation: `get`, which the user can alias into something like `T`.
|
||||
|
||||
`get`, referred to as `T` here, has a few different signatures.
|
||||
All of them will return the key if the entry can't be found in the active translation catalog.
|
||||
|
||||
- `T(key)` returns the translation of `key`.
|
||||
- `T(key, n)` returns a pluralized translation of `key` according to value `n`.
|
||||
|
||||
- `T(section, key)` returns the translation of `key` in `section`.
|
||||
- `T(section, key, n)` returns a pluralized translation of `key` in `section` according to value `n`.
|
||||
|
||||
By default lookup take place in the global `i18n.ACTIVE` catalog for ease of use.
|
||||
If you want to override which translation to use, for example in a language preview dialog, you can use the following:
|
||||
|
||||
- `T(key, n, catalog)` returns the pluralized version of `key` from explictly supplied catalog.
|
||||
- `T(section, key, n, catalog)` returns the pluralized version of `key` in `section` from explictly supplied catalog.
|
||||
|
||||
If a catalog has translation contexts or sections, then ommitting it in the above calls looks up in section "".
|
||||
|
||||
The default pluralization rule is n != 1, which is to say that passing n == 1 (or not passing n) returns the singular form.
|
||||
Passing n != 1 returns plural form 1.
|
||||
|
||||
Should a language not conform to this rule, you can pass a pluralizer procedure to the catalog parser.
|
||||
This is a procedure that maps an integer to an integer, taking a value and returning which plural slot should be used.
|
||||
|
||||
You can also assign it to a loaded catalog after parsing, of course.
|
||||
|
||||
Some code examples follow.
|
||||
*/
|
||||
|
||||
/*
|
||||
```cpp
|
||||
import "core:fmt"
|
||||
import "core:text/i18n"
|
||||
|
||||
T :: i18n.get
|
||||
|
||||
mo :: proc() {
|
||||
using fmt
|
||||
|
||||
err: i18n.Error
|
||||
|
||||
/*
|
||||
Parse MO file and set it as the active translation so we can omit `get`'s "catalog" parameter.
|
||||
*/
|
||||
i18n.ACTIVE, err = i18n.parse_mo(#load("translations/nl_NL.mo"))
|
||||
defer i18n.destroy()
|
||||
|
||||
if err != .None { return }
|
||||
|
||||
/*
|
||||
These are in the .MO catalog.
|
||||
*/
|
||||
println("-----")
|
||||
println(T(""))
|
||||
println("-----")
|
||||
println(T("There are 69,105 leaves here."))
|
||||
println("-----")
|
||||
println(T("Hellope, World!"))
|
||||
println("-----")
|
||||
// We pass 1 into `T` to get the singular format string, then 1 again into printf.
|
||||
printf(T("There is %d leaf.\n", 1), 1)
|
||||
// We pass 42 into `T` to get the plural format string, then 42 again into printf.
|
||||
printf(T("There is %d leaf.\n", 42), 42)
|
||||
|
||||
/*
|
||||
This isn't in the translation catalog, so the key is passed back untranslated.
|
||||
*/
|
||||
println("-----")
|
||||
println(T("Come visit us on Discord!"))
|
||||
}
|
||||
|
||||
qt :: proc() {
|
||||
using fmt
|
||||
|
||||
err: i18n.Error
|
||||
|
||||
/*
|
||||
Parse QT file and set it as the active translation so we can omit `get`'s "catalog" parameter.
|
||||
*/
|
||||
i18n.ACTIVE, err = i18n.parse_qt(#load("translations/nl_NL-qt-ts.ts"))
|
||||
defer i18n.destroy()
|
||||
|
||||
if err != .None {
|
||||
return
|
||||
}
|
||||
|
||||
/*
|
||||
These are in the .TS catalog. As you can see they have sections.
|
||||
*/
|
||||
println("--- Page section ---")
|
||||
println("Page:Text for translation =", T("Page", "Text for translation"))
|
||||
println("-----")
|
||||
println("Page:Also text to translate =", T("Page", "Also text to translate"))
|
||||
println("-----")
|
||||
println("--- installscript section ---")
|
||||
println("installscript:99 bottles of beer on the wall =", T("installscript", "99 bottles of beer on the wall"))
|
||||
println("-----")
|
||||
println("--- apple_count section ---")
|
||||
println("apple_count:%d apple(s) =")
|
||||
println("\t 1 =", T("apple_count", "%d apple(s)", 1))
|
||||
println("\t 42 =", T("apple_count", "%d apple(s)", 42))
|
||||
}
|
||||
```
|
||||
*/
|
||||
@@ -0,0 +1,167 @@
|
||||
package i18n
|
||||
/*
|
||||
A parser for GNU GetText .MO files.
|
||||
|
||||
Copyright 2021-2022 Jeroen van Rijn <nom@duclavier.com>.
|
||||
Made available under Odin's BSD-3 license.
|
||||
|
||||
A from-scratch implementation based after the specification found here:
|
||||
https://www.gnu.org/software/gettext/manual/html_node/MO-Files.html
|
||||
|
||||
Options are ignored as they're not applicable to this format.
|
||||
They're part of the signature for consistency with other catalog formats.
|
||||
|
||||
List of contributors:
|
||||
Jeroen van Rijn: Initial implementation.
|
||||
*/
|
||||
import "core:os"
|
||||
import "core:strings"
|
||||
import "core:bytes"
|
||||
|
||||
parse_mo_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
|
||||
context.allocator = allocator
|
||||
/*
|
||||
An MO file should have at least a 4-byte magic, 2 x 2 byte version info,
|
||||
a 4-byte number of strings value, and 2 x 4-byte offsets.
|
||||
*/
|
||||
if len(data) < 20 {
|
||||
return {}, .MO_File_Invalid
|
||||
}
|
||||
|
||||
/*
|
||||
Check magic. Should be 0x950412de in native Endianness.
|
||||
*/
|
||||
native := true
|
||||
magic := read_u32(data, native) or_return
|
||||
|
||||
if magic != 0x950412de {
|
||||
native = false
|
||||
magic = read_u32(data, native) or_return
|
||||
|
||||
if magic != 0x950412de { return {}, .MO_File_Invalid_Signature }
|
||||
}
|
||||
|
||||
/*
|
||||
We can ignore version_minor at offset 6.
|
||||
*/
|
||||
version_major := read_u16(data[4:]) or_return
|
||||
if version_major > 1 { return {}, .MO_File_Unsupported_Version }
|
||||
|
||||
count := read_u32(data[ 8:]) or_return
|
||||
original_offset := read_u32(data[12:]) or_return
|
||||
translated_offset := read_u32(data[16:]) or_return
|
||||
|
||||
if count == 0 { return {}, .Empty_Translation_Catalog }
|
||||
|
||||
/*
|
||||
Initalize Translation, interner and optional pluralizer.
|
||||
*/
|
||||
translation = new(Translation)
|
||||
translation.pluralize = pluralizer
|
||||
strings.intern_init(&translation.intern, allocator, allocator)
|
||||
|
||||
// Gettext MO files only have one section.
|
||||
translation.k_v[""] = {}
|
||||
section := &translation.k_v[""]
|
||||
|
||||
for n := u32(0); n < count; n += 1 {
|
||||
/*
|
||||
Grab string's original length and offset.
|
||||
*/
|
||||
offset := original_offset + 8 * n
|
||||
if len(data) < int(offset + 8) { return translation, .MO_File_Invalid }
|
||||
|
||||
o_length := read_u32(data[offset :], native) or_return
|
||||
o_offset := read_u32(data[offset + 4:], native) or_return
|
||||
|
||||
offset = translated_offset + 8 * n
|
||||
if len(data) < int(offset + 8) { return translation, .MO_File_Invalid }
|
||||
|
||||
t_length := read_u32(data[offset :], native) or_return
|
||||
t_offset := read_u32(data[offset + 4:], native) or_return
|
||||
|
||||
max_offset := int(max(o_offset + o_length + 1, t_offset + t_length + 1))
|
||||
if len(data) < max_offset { return translation, .Premature_EOF }
|
||||
|
||||
key := data[o_offset:][:o_length]
|
||||
val := data[t_offset:][:t_length]
|
||||
|
||||
/*
|
||||
Could be a pluralized string.
|
||||
*/
|
||||
zero := []byte{0}
|
||||
|
||||
keys := bytes.split(key, zero)
|
||||
vals := bytes.split(val, zero)
|
||||
|
||||
if len(keys) != len(vals) || max(len(keys), len(vals)) > MAX_PLURALS {
|
||||
return translation, .MO_File_Incorrect_Plural_Count
|
||||
}
|
||||
|
||||
for k in keys {
|
||||
interned_key := strings.intern_get(&translation.intern, string(k))
|
||||
|
||||
interned_vals := make([]string, len(keys))
|
||||
last_val: string
|
||||
|
||||
i := 0
|
||||
for v in vals {
|
||||
interned_vals[i] = strings.intern_get(&translation.intern, string(v))
|
||||
last_val = interned_vals[i]
|
||||
i += 1
|
||||
}
|
||||
section[interned_key] = interned_vals
|
||||
}
|
||||
delete(vals)
|
||||
delete(keys)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
parse_mo_file :: proc(filename: string, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
|
||||
context.allocator = allocator
|
||||
|
||||
data, data_ok := os.read_entire_file(filename)
|
||||
defer delete(data)
|
||||
|
||||
if !data_ok { return {}, .File_Error }
|
||||
|
||||
return parse_mo_from_slice(data, options, pluralizer, allocator)
|
||||
}
|
||||
|
||||
parse_mo :: proc { parse_mo_file, parse_mo_from_slice }
|
||||
|
||||
/*
|
||||
Helpers.
|
||||
*/
|
||||
read_u32 :: proc(data: []u8, native_endian := true) -> (res: u32, err: Error) {
|
||||
if len(data) < size_of(u32) { return 0, .Premature_EOF }
|
||||
|
||||
val := (^u32)(raw_data(data))^
|
||||
|
||||
if native_endian {
|
||||
return val, .None
|
||||
} else {
|
||||
when ODIN_ENDIAN == .Little {
|
||||
return u32(transmute(u32be)val), .None
|
||||
} else {
|
||||
return u32(transmute(u32le)val), .None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
read_u16 :: proc(data: []u8, native_endian := true) -> (res: u16, err: Error) {
|
||||
if len(data) < size_of(u16) { return 0, .Premature_EOF }
|
||||
|
||||
val := (^u16)(raw_data(data))^
|
||||
|
||||
if native_endian {
|
||||
return val, .None
|
||||
} else {
|
||||
when ODIN_ENDIAN == .Little {
|
||||
return u16(transmute(u16be)val), .None
|
||||
} else {
|
||||
return u16(transmute(u16le)val), .None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package i18n
|
||||
/*
|
||||
Internationalization helpers.
|
||||
|
||||
Copyright 2021-2022 Jeroen van Rijn <nom@duclavier.com>.
|
||||
Made available under Odin's BSD-3 license.
|
||||
|
||||
List of contributors:
|
||||
Jeroen van Rijn: Initial implementation.
|
||||
*/
|
||||
import "core:strings"
|
||||
|
||||
/*
|
||||
TODO:
|
||||
- Support for more translation catalog file formats.
|
||||
*/
|
||||
|
||||
/*
|
||||
Currently active catalog.
|
||||
*/
|
||||
ACTIVE: ^Translation
|
||||
|
||||
// Allow between 1 and 255 plural forms. Default: 10.
|
||||
MAX_PLURALS :: min(max(#config(ODIN_i18N_MAX_PLURAL_FORMS, 10), 1), 255)
|
||||
|
||||
/*
|
||||
The main data structure. This can be generated from various different file formats, as long as we have a parser for them.
|
||||
*/
|
||||
|
||||
Section :: map[string][]string
|
||||
|
||||
Translation :: struct {
|
||||
k_v: map[string]Section, // k_v[section][key][plural_form] = ...
|
||||
intern: strings.Intern,
|
||||
|
||||
pluralize: proc(number: int) -> int,
|
||||
}
|
||||
|
||||
Error :: enum {
|
||||
/*
|
||||
General return values.
|
||||
*/
|
||||
None = 0,
|
||||
Empty_Translation_Catalog,
|
||||
Duplicate_Key,
|
||||
|
||||
/*
|
||||
Couldn't find, open or read file.
|
||||
*/
|
||||
File_Error,
|
||||
|
||||
/*
|
||||
File too short.
|
||||
*/
|
||||
Premature_EOF,
|
||||
|
||||
/*
|
||||
GNU Gettext *.MO file errors.
|
||||
*/
|
||||
MO_File_Invalid_Signature,
|
||||
MO_File_Unsupported_Version,
|
||||
MO_File_Invalid,
|
||||
MO_File_Incorrect_Plural_Count,
|
||||
|
||||
/*
|
||||
Qt Linguist *.TS file errors.
|
||||
*/
|
||||
TS_File_Parse_Error,
|
||||
TS_File_Expected_Context,
|
||||
TS_File_Expected_Context_Name,
|
||||
TS_File_Expected_Source,
|
||||
TS_File_Expected_Translation,
|
||||
TS_File_Expected_NumerusForm,
|
||||
|
||||
}
|
||||
|
||||
Parse_Options :: struct {
|
||||
merge_sections: bool,
|
||||
}
|
||||
|
||||
DEFAULT_PARSE_OPTIONS :: Parse_Options{
|
||||
merge_sections = false,
|
||||
}
|
||||
|
||||
/*
|
||||
Several ways to use:
|
||||
- get(key), which defaults to the singular form and i18n.ACTIVE catalog, or
|
||||
- get(key, number), which returns the appropriate plural from the active catalog, or
|
||||
- get(key, number, catalog) to grab text from a specific one.
|
||||
*/
|
||||
get_single_section :: proc(key: string, number := 0, catalog: ^Translation = ACTIVE) -> (value: string) {
|
||||
/*
|
||||
A lot of languages use singular for 1 item and plural for 0 or more than 1 items. This is our default pluralize rule.
|
||||
*/
|
||||
plural := 1 if number != 1 else 0
|
||||
|
||||
if catalog.pluralize != nil {
|
||||
plural = catalog.pluralize(number)
|
||||
}
|
||||
return get_by_slot(key, plural, catalog)
|
||||
}
|
||||
|
||||
/*
|
||||
Several ways to use:
|
||||
- get(section, key), which defaults to the singular form and i18n.ACTIVE catalog, or
|
||||
- get(section, key, number), which returns the appropriate plural from the active catalog, or
|
||||
- get(section, key, number, catalog) to grab text from a specific one.
|
||||
*/
|
||||
get_by_section :: proc(section, key: string, number := 0, catalog: ^Translation = ACTIVE) -> (value: string) {
|
||||
/*
|
||||
A lot of languages use singular for 1 item and plural for 0 or more than 1 items. This is our default pluralize rule.
|
||||
*/
|
||||
plural := 1 if number != 1 else 0
|
||||
|
||||
if catalog.pluralize != nil {
|
||||
plural = catalog.pluralize(number)
|
||||
}
|
||||
return get_by_slot(section, key, plural, catalog)
|
||||
}
|
||||
get :: proc{get_single_section, get_by_section}
|
||||
|
||||
/*
|
||||
Several ways to use:
|
||||
- get_by_slot(key), which defaults to the singular form and i18n.ACTIVE catalog, or
|
||||
- get_by_slot(key, slot), which returns the requested plural from the active catalog, or
|
||||
- get_by_slot(key, slot, catalog) to grab text from a specific one.
|
||||
|
||||
If a file format parser doesn't (yet) support plural slots, each of the slots will point at the same string.
|
||||
*/
|
||||
get_by_slot_single_section :: proc(key: string, slot := 0, catalog: ^Translation = ACTIVE) -> (value: string) {
|
||||
return get_by_slot_by_section("", key, slot, catalog)
|
||||
}
|
||||
|
||||
/*
|
||||
Several ways to use:
|
||||
- get_by_slot(key), which defaults to the singular form and i18n.ACTIVE catalog, or
|
||||
- get_by_slot(key, slot), which returns the requested plural from the active catalog, or
|
||||
- get_by_slot(key, slot, catalog) to grab text from a specific one.
|
||||
|
||||
If a file format parser doesn't (yet) support plural slots, each of the slots will point at the same string.
|
||||
*/
|
||||
get_by_slot_by_section :: proc(section, key: string, slot := 0, catalog: ^Translation = ACTIVE) -> (value: string) {
|
||||
if catalog == nil || section not_in catalog.k_v {
|
||||
/*
|
||||
Return the key if the catalog catalog hasn't been initialized yet, or the section is not present.
|
||||
*/
|
||||
return key
|
||||
}
|
||||
|
||||
/*
|
||||
Return the translation from the requested slot if this key is known, else return the key.
|
||||
*/
|
||||
if translations, ok := catalog.k_v[section][key]; ok {
|
||||
plural := min(max(0, slot), len(catalog.k_v[section][key]) - 1)
|
||||
return translations[plural]
|
||||
}
|
||||
return key
|
||||
}
|
||||
get_by_slot :: proc{get_by_slot_single_section, get_by_slot_by_section}
|
||||
|
||||
/*
|
||||
Same for destroy:
|
||||
- destroy(), to clean up the currently active catalog catalog i18n.ACTIVE
|
||||
- destroy(catalog), to clean up a specific catalog.
|
||||
*/
|
||||
destroy :: proc(catalog: ^Translation = ACTIVE, allocator := context.allocator) {
|
||||
context.allocator = allocator
|
||||
|
||||
if catalog == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for section in &catalog.k_v {
|
||||
for key in &catalog.k_v[section] {
|
||||
delete(catalog.k_v[section][key])
|
||||
}
|
||||
delete(catalog.k_v[section])
|
||||
}
|
||||
delete(catalog.k_v)
|
||||
strings.intern_destroy(&catalog.intern)
|
||||
free(catalog)
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package i18n
|
||||
/*
|
||||
A parser for Qt Linguist TS files.
|
||||
|
||||
Copyright 2022 Jeroen van Rijn <nom@duclavier.com>.
|
||||
Made available under Odin's BSD-3 license.
|
||||
|
||||
A from-scratch implementation based after the specification found here:
|
||||
https://doc.qt.io/qt-5/linguist-ts-file-format.html
|
||||
|
||||
List of contributors:
|
||||
Jeroen van Rijn: Initial implementation.
|
||||
*/
|
||||
import "core:os"
|
||||
import "core:encoding/xml"
|
||||
import "core:strings"
|
||||
|
||||
TS_XML_Options := xml.Options{
|
||||
flags = {
|
||||
.Input_May_Be_Modified,
|
||||
.Must_Have_Prolog,
|
||||
.Must_Have_DocType,
|
||||
.Ignore_Unsupported,
|
||||
.Unbox_CDATA,
|
||||
.Decode_SGML_Entities,
|
||||
},
|
||||
expected_doctype = "TS",
|
||||
}
|
||||
|
||||
parse_qt_linguist_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
|
||||
context.allocator = allocator
|
||||
|
||||
ts, xml_err := xml.parse(data, TS_XML_Options)
|
||||
defer xml.destroy(ts)
|
||||
|
||||
if xml_err != .None || ts.element_count < 1 || ts.elements[0].ident != "TS" || len(ts.elements[0].children) == 0 {
|
||||
return nil, .TS_File_Parse_Error
|
||||
}
|
||||
|
||||
/*
|
||||
Initalize Translation, interner and optional pluralizer.
|
||||
*/
|
||||
translation = new(Translation)
|
||||
translation.pluralize = pluralizer
|
||||
strings.intern_init(&translation.intern, allocator, allocator)
|
||||
|
||||
section: ^Section
|
||||
|
||||
for child_id in ts.elements[0].children {
|
||||
// These should be <context>s.
|
||||
child := ts.elements[child_id]
|
||||
if child.ident != "context" {
|
||||
return translation, .TS_File_Expected_Context
|
||||
}
|
||||
|
||||
// Find section name.
|
||||
section_name_id, section_name_found := xml.find_child_by_ident(ts, child_id, "name")
|
||||
if !section_name_found {
|
||||
return translation, .TS_File_Expected_Context_Name,
|
||||
}
|
||||
|
||||
section_name := strings.intern_get(&translation.intern, "")
|
||||
if !options.merge_sections {
|
||||
section_name = strings.intern_get(&translation.intern, ts.elements[section_name_id].value)
|
||||
}
|
||||
|
||||
if section_name not_in translation.k_v {
|
||||
translation.k_v[section_name] = {}
|
||||
}
|
||||
section = &translation.k_v[section_name]
|
||||
|
||||
// Find messages in section.
|
||||
nth: int
|
||||
for {
|
||||
message_id, message_found := xml.find_child_by_ident(ts, child_id, "message", nth)
|
||||
if !message_found {
|
||||
break
|
||||
}
|
||||
|
||||
numerus_tag, _ := xml.find_attribute_val_by_key(ts, message_id, "numerus")
|
||||
has_plurals := numerus_tag == "yes"
|
||||
|
||||
// We must have a <source> = key
|
||||
source_id, source_found := xml.find_child_by_ident(ts, message_id, "source")
|
||||
if !source_found {
|
||||
return translation, .TS_File_Expected_Source
|
||||
}
|
||||
|
||||
// We must have a <translation>
|
||||
translation_id, translation_found := xml.find_child_by_ident(ts, message_id, "translation")
|
||||
if !translation_found {
|
||||
return translation, .TS_File_Expected_Translation
|
||||
}
|
||||
|
||||
source := strings.intern_get(&translation.intern, ts.elements[source_id].value)
|
||||
xlat := strings.intern_get(&translation.intern, ts.elements[translation_id].value)
|
||||
|
||||
if source in section {
|
||||
return translation, .Duplicate_Key
|
||||
}
|
||||
|
||||
if has_plurals {
|
||||
if xlat != "" {
|
||||
return translation, .TS_File_Expected_NumerusForm
|
||||
}
|
||||
|
||||
num_plurals: int
|
||||
for {
|
||||
numerus_id, numerus_found := xml.find_child_by_ident(ts, translation_id, "numerusform", num_plurals)
|
||||
if !numerus_found {
|
||||
break
|
||||
}
|
||||
num_plurals += 1
|
||||
}
|
||||
|
||||
if num_plurals < 2 {
|
||||
return translation, .TS_File_Expected_NumerusForm
|
||||
}
|
||||
section[source] = make([]string, num_plurals)
|
||||
|
||||
num_plurals = 0
|
||||
for {
|
||||
numerus_id, numerus_found := xml.find_child_by_ident(ts, translation_id, "numerusform", num_plurals)
|
||||
if !numerus_found {
|
||||
break
|
||||
}
|
||||
numerus := strings.intern_get(&translation.intern, ts.elements[numerus_id].value)
|
||||
section[source][num_plurals] = numerus
|
||||
|
||||
num_plurals += 1
|
||||
}
|
||||
} else {
|
||||
// Single translation
|
||||
section[source] = make([]string, 1)
|
||||
section[source][0] = xlat
|
||||
}
|
||||
|
||||
nth += 1
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
parse_qt_linguist_file :: proc(filename: string, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
|
||||
context.allocator = allocator
|
||||
|
||||
data, data_ok := os.read_entire_file(filename)
|
||||
defer delete(data)
|
||||
|
||||
if !data_ok { return {}, .File_Error }
|
||||
|
||||
return parse_qt_linguist_from_slice(data, options, pluralizer, allocator)
|
||||
}
|
||||
|
||||
parse_qt :: proc { parse_qt_linguist_file, parse_qt_linguist_from_slice }
|
||||
@@ -44,10 +44,10 @@ Pool :: struct {
|
||||
}
|
||||
|
||||
// Once initialized, the pool's memory address is not allowed to change until
|
||||
// it is destroyed. If thread_count < 1, thread count 1 will be used.
|
||||
// it is destroyed.
|
||||
//
|
||||
// The thread pool requires an allocator which it either owns, or which is thread safe.
|
||||
pool_init :: proc(pool: ^Pool, thread_count: int, allocator: mem.Allocator) {
|
||||
pool_init :: proc(pool: ^Pool, allocator: mem.Allocator, thread_count: int) {
|
||||
context.allocator = allocator
|
||||
pool.allocator = allocator
|
||||
pool.tasks = make([dynamic]Task)
|
||||
@@ -113,9 +113,8 @@ pool_join :: proc(pool: ^Pool) {
|
||||
// the thread pool. You can even add tasks from inside other tasks.
|
||||
//
|
||||
// Each task also needs an allocator which it either owns, or which is thread
|
||||
// safe. By default, allocations in the task are disabled by use of the
|
||||
// nil_allocator.
|
||||
pool_add_task :: proc(pool: ^Pool, procedure: Task_Proc, data: rawptr, user_index: int = 0, allocator := context.allocator) {
|
||||
// safe.
|
||||
pool_add_task :: proc(pool: ^Pool, allocator: mem.Allocator, procedure: Task_Proc, data: rawptr, user_index: int = 0) {
|
||||
sync.guard(&pool.mutex)
|
||||
|
||||
append(&pool.tasks, Task{
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
package xml_example
|
||||
|
||||
import "core:encoding/xml"
|
||||
import "core:os"
|
||||
import "core:path"
|
||||
import "core:mem"
|
||||
import "core:strings"
|
||||
import "core:strconv"
|
||||
import "core:slice"
|
||||
import "core:fmt"
|
||||
|
||||
/*
|
||||
Silent error handler for the parser.
|
||||
*/
|
||||
Error_Handler :: proc(pos: xml.Pos, fmt: string, args: ..any) {}
|
||||
|
||||
OPTIONS :: xml.Options{ flags = { .Ignore_Unsupported, }, expected_doctype = "unicode", }
|
||||
|
||||
Entity :: struct {
|
||||
name: string,
|
||||
codepoint: rune,
|
||||
description: string,
|
||||
}
|
||||
|
||||
generate_encoding_entity_table :: proc() {
|
||||
using fmt
|
||||
|
||||
filename := path.join(ODIN_ROOT, "tests", "core", "assets", "XML", "unicode.xml")
|
||||
defer delete(filename)
|
||||
|
||||
generated_filename := path.join(ODIN_ROOT, "core", "encoding", "entity", "generated.odin")
|
||||
defer delete(generated_filename)
|
||||
|
||||
doc, err := xml.parse(filename, OPTIONS, Error_Handler)
|
||||
defer xml.destroy(doc)
|
||||
|
||||
if err != .None {
|
||||
printf("Load/Parse error: %v\n", err)
|
||||
if err == .File_Error {
|
||||
printf("\"%v\" not found. Did you run \"tests\\download_assets.py\"?", filename)
|
||||
}
|
||||
os.exit(1)
|
||||
}
|
||||
|
||||
printf("\"%v\" loaded and parsed.\n", filename)
|
||||
|
||||
generated_buf: strings.Builder
|
||||
defer strings.destroy_builder(&generated_buf)
|
||||
w := strings.to_writer(&generated_buf)
|
||||
|
||||
charlist, charlist_ok := xml.find_child_by_ident(doc.root, "charlist")
|
||||
if !charlist_ok {
|
||||
eprintln("Could not locate top-level `<charlist>` tag.")
|
||||
os.exit(1)
|
||||
}
|
||||
|
||||
printf("Found `<charlist>` with %v children.\n", len(charlist.children))
|
||||
|
||||
entity_map: map[string]Entity
|
||||
names: [dynamic]string
|
||||
|
||||
min_name_length := max(int)
|
||||
max_name_length := min(int)
|
||||
shortest_name: string
|
||||
longest_name: string
|
||||
|
||||
count := 0
|
||||
for char in charlist.children {
|
||||
if char.ident != "character" {
|
||||
eprintf("Expected `<character>`, got `<%v>`\n", char.ident)
|
||||
os.exit(1)
|
||||
}
|
||||
|
||||
if codepoint_string, ok := xml.find_attribute_val_by_key(char, "dec"); !ok {
|
||||
eprintln("`<character id=\"...\">` attribute not found.")
|
||||
os.exit(1)
|
||||
} else {
|
||||
codepoint := strconv.atoi(codepoint_string)
|
||||
|
||||
desc, desc_ok := xml.find_child_by_ident(char, "description")
|
||||
description := desc.value if desc_ok else ""
|
||||
|
||||
/*
|
||||
For us to be interested in this codepoint, it has to have at least one entity.
|
||||
*/
|
||||
|
||||
nth := 0
|
||||
for {
|
||||
character_entity, entity_ok := xml.find_child_by_ident(char, "entity", nth)
|
||||
if !entity_ok { break }
|
||||
|
||||
nth += 1
|
||||
if name, name_ok := xml.find_attribute_val_by_key(character_entity, "id"); name_ok {
|
||||
|
||||
if len(name) == 0 {
|
||||
/*
|
||||
Invalid name. Skip.
|
||||
*/
|
||||
continue
|
||||
}
|
||||
|
||||
if name == "\"\"" {
|
||||
printf("%#v\n", char)
|
||||
printf("%#v\n", character_entity)
|
||||
}
|
||||
|
||||
if len(name) > max_name_length { longest_name = name }
|
||||
if len(name) < min_name_length { shortest_name = name }
|
||||
|
||||
min_name_length = min(min_name_length, len(name))
|
||||
max_name_length = max(max_name_length, len(name))
|
||||
|
||||
e := Entity{
|
||||
name = name,
|
||||
codepoint = rune(codepoint),
|
||||
description = description,
|
||||
}
|
||||
|
||||
if _, seen := entity_map[name]; seen {
|
||||
continue
|
||||
}
|
||||
|
||||
entity_map[name] = e
|
||||
append(&names, name)
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Sort by name.
|
||||
*/
|
||||
slice.sort(names[:])
|
||||
|
||||
printf("Found %v unique `&name;` -> rune mappings.\n", count)
|
||||
printf("Shortest name: %v (%v)\n", shortest_name, min_name_length)
|
||||
printf("Longest name: %v (%v)\n", longest_name, max_name_length)
|
||||
|
||||
// println(rune_to_string(1234))
|
||||
|
||||
/*
|
||||
Generate table.
|
||||
*/
|
||||
wprintln(w, "package unicode_entity")
|
||||
wprintln(w, "")
|
||||
wprintln(w, GENERATED)
|
||||
wprintln(w, "")
|
||||
wprintf (w, TABLE_FILE_PROLOG)
|
||||
wprintln(w, "")
|
||||
|
||||
wprintf (w, "// `&%v;`\n", shortest_name)
|
||||
wprintf (w, "XML_NAME_TO_RUNE_MIN_LENGTH :: %v\n", min_name_length)
|
||||
wprintf (w, "// `&%v;`\n", longest_name)
|
||||
wprintf (w, "XML_NAME_TO_RUNE_MAX_LENGTH :: %v\n", max_name_length)
|
||||
wprintln(w, "")
|
||||
|
||||
wprintln(w,
|
||||
`
|
||||
/*
|
||||
Input:
|
||||
entity_name - a string, like "copy" that describes a user-encoded Unicode entity as used in XML.
|
||||
|
||||
Output:
|
||||
"decoded" - The decoded rune if found by name, or -1 otherwise.
|
||||
"ok" - true if found, false if not.
|
||||
|
||||
IMPORTANT: XML processors (including browsers) treat these names as case-sensitive. So do we.
|
||||
*/
|
||||
named_xml_entity_to_rune :: proc(name: string) -> (decoded: rune, ok: bool) {
|
||||
/*
|
||||
Early out if the name is too short or too long.
|
||||
min as a precaution in case the generated table has a bogus value.
|
||||
*/
|
||||
if len(name) < min(1, XML_NAME_TO_RUNE_MIN_LENGTH) || len(name) > XML_NAME_TO_RUNE_MAX_LENGTH {
|
||||
return -1, false
|
||||
}
|
||||
|
||||
switch rune(name[0]) {
|
||||
`)
|
||||
|
||||
prefix := '?'
|
||||
should_close := false
|
||||
|
||||
for v in names {
|
||||
if rune(v[0]) != prefix {
|
||||
if should_close {
|
||||
wprintln(w, "\t\t}\n")
|
||||
}
|
||||
|
||||
prefix = rune(v[0])
|
||||
wprintf (w, "\tcase '%v':\n", prefix)
|
||||
wprintln(w, "\t\tswitch name {")
|
||||
}
|
||||
|
||||
e := entity_map[v]
|
||||
|
||||
wprintf(w, "\t\t\tcase \"%v\": \n", e.name)
|
||||
wprintf(w, "\t\t\t\t// %v\n", e.description)
|
||||
wprintf(w, "\t\t\t\treturn %v, true\n", rune_to_string(e.codepoint))
|
||||
|
||||
should_close = true
|
||||
}
|
||||
wprintln(w, "\t\t}")
|
||||
wprintln(w, "\t}")
|
||||
wprintln(w, "\treturn -1, false")
|
||||
wprintln(w, "}\n")
|
||||
wprintln(w, GENERATED)
|
||||
|
||||
println()
|
||||
println(strings.to_string(generated_buf))
|
||||
println()
|
||||
|
||||
written := os.write_entire_file(generated_filename, transmute([]byte)strings.to_string(generated_buf))
|
||||
|
||||
if written {
|
||||
fmt.printf("Successfully written generated \"%v\".", generated_filename)
|
||||
} else {
|
||||
fmt.printf("Failed to write generated \"%v\".", generated_filename)
|
||||
}
|
||||
|
||||
delete(entity_map)
|
||||
delete(names)
|
||||
for name in &names {
|
||||
free(&name)
|
||||
}
|
||||
}
|
||||
|
||||
GENERATED :: `/*
|
||||
------ GENERATED ------ DO NOT EDIT ------ GENERATED ------ DO NOT EDIT ------ GENERATED ------
|
||||
*/`
|
||||
|
||||
TABLE_FILE_PROLOG :: `/*
|
||||
This file is generated from "https://www.w3.org/2003/entities/2007xml/unicode.xml".
|
||||
|
||||
UPDATE:
|
||||
- Ensure the XML file was downloaded using "tests\core\download_assets.py".
|
||||
- Run "core/unicode/tools/generate_entity_table.odin"
|
||||
|
||||
Odin unicode generated tables: https://github.com/odin-lang/Odin/tree/master/core/encoding/entity
|
||||
|
||||
Copyright © 2021 World Wide Web Consortium, (Massachusetts Institute of Technology,
|
||||
European Research Consortium for Informatics and Mathematics, Keio University, Beihang).
|
||||
|
||||
All Rights Reserved.
|
||||
|
||||
This work is distributed under the W3C® Software License [1] in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
||||
|
||||
[1] http://www.w3.org/Consortium/Legal/copyright-software
|
||||
|
||||
See also: LICENSE_table.md
|
||||
*/
|
||||
`
|
||||
|
||||
rune_to_string :: proc(r: rune) -> (res: string) {
|
||||
res = fmt.tprintf("%08x", int(r))
|
||||
for len(res) > 2 && res[:2] == "00" {
|
||||
res = res[2:]
|
||||
}
|
||||
return fmt.tprintf("rune(0x%v)", res)
|
||||
}
|
||||
|
||||
is_dotted_name :: proc(name: string) -> (dotted: bool) {
|
||||
for r in name {
|
||||
if r == '.' { return true}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
main :: proc() {
|
||||
using fmt
|
||||
|
||||
track: mem.Tracking_Allocator
|
||||
mem.tracking_allocator_init(&track, context.allocator)
|
||||
context.allocator = mem.tracking_allocator(&track)
|
||||
|
||||
generate_encoding_entity_table()
|
||||
|
||||
if len(track.allocation_map) > 0 {
|
||||
println()
|
||||
for _, v in track.allocation_map {
|
||||
printf("%v Leaked %v bytes.\n", v.location, v.size)
|
||||
}
|
||||
}
|
||||
println("Done and cleaned up!")
|
||||
}
|
||||
Reference in New Issue
Block a user