lexer fixes; eval2 resolution qualifiers

project.4coder

@@ -49,7 +49,7 @@ commands =
   // .f1 = { .win = "raddbg_stable --ipc kill_all && build raddbg", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, },
   // .f1 = { .win = "raddbg_stable --ipc kill_all && build raddbg debug telemetry", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, },
   // .f1 = { .win = "raddbg_stable --ipc kill_all && build radbin", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, },
-  .f1 = { .win = "raddbg_stable --ipc kill_all && build raddbg", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, },
+  .f1 = { .win = "raddbg_stable --ipc kill_all && build torture && build ryan_scratch && pushd build && torture eval2/* && ryan_scratch", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, },
   
   //- rjf: [raddbg wsl]
   // .f1 = { .win = "wsl ./build.sh raddbg", .linux = "", .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, .cursor_at_end = false, },

project.raddbg

@@ -26,7 +26,7 @@ target:
 {
   executable: "build/torture.exe"
   working_directory: build
-  arguments: "raddbg/*"
+  arguments: "eval2/*"
 }
 target:
 {

src/eval2/eval2.c

@@ -1917,12 +1917,46 @@ e2_parse_from_string(Arena *arena, E2_ParseState *state, B32 identifier_is_type,
     //- rjf: standalone identifiers (also could be definition or operator keyword)
     else if(need_new_expr && e2_try_token(lang, string, E2_TokenKind_Identifier, s(""), &off, &token))
     {
-      String8 identifier = str8_substr(string, token.range);
       B32 identifier_mapped = 0;
       B32 definitions_are_allowed = (state->top_task->expr_kind != E2_ExprKind_Macro);
       
+      // rjf: pick the last identifier as the target identifier, collect the rest as qualifiers
+      String8List qualifiers = {0};
+      String8 identifier = {0};
+      {
+        E2_Token last_identifier_token = token;
+        for(;;)
+        {
+          U64 next_off_maybe = off;
+          E2_Token ext_identifier_token = {E2_TokenKind_Null};
+          if(e2_try_token(lang, string, E2_TokenKind_Symbol, s("!"), &next_off_maybe, 0) ||
+             (e2_try_token(lang, string, E2_TokenKind_Symbol, s("."), &next_off_maybe, 0) &&
+              e2_try_token(lang, string, E2_TokenKind_Identifier, s(""), &next_off_maybe, &ext_identifier_token) &&
+              e2_try_token(lang, string, E2_TokenKind_Symbol, s("!"), &next_off_maybe, 0)))
+          {
+            Rng1U64 qualifier_range = last_identifier_token.range;
+            if(dim_1u64(ext_identifier_token.range) > 0)
+            {
+              qualifier_range = union_1u64(qualifier_range, ext_identifier_token.range);
+            }
+            String8 qualifier = str8_substr(string, qualifier_range);
+            str8_list_push(arena, &qualifiers, qualifier);
+            if(!e2_try_token(lang, string, E2_TokenKind_Identifier, s(""), &next_off_maybe, &last_identifier_token))
+            {
+              e2_msgf(arena, &parse.msgs, r1u64(off, off), "Expected identifier after qualifier `%S!`.", qualifier);
+            }
+            off = next_off_maybe;
+          }
+          else
+          {
+            identifier = str8_substr(string, last_identifier_token.range);
+            break;
+          }
+        }
+      }
+      
       // rjf: first try to resolve as a definition
-      if(definitions_are_allowed && !identifier_mapped && e2_try_token(lang, string, E2_TokenKind_Symbol, s("="), &off, 0))
+      if(qualifiers.node_count == 0 && definitions_are_allowed && !identifier_mapped && e2_try_token(lang, string, E2_TokenKind_Symbol, s("="), &off, 0))
       {
         identifier_mapped = 1;
         E2_ParseTask *task = state->free_task;
@@ -1944,7 +1978,7 @@ e2_parse_from_string(Arena *arena, E2_ParseState *state, B32 identifier_is_type,
       }
       
       // rjf: try to resolve it as a macro definition
-      if(definitions_are_allowed && !identifier_mapped)
+      if(qualifiers.node_count == 0 && definitions_are_allowed && !identifier_mapped)
       {
         U64 macro_off = off;
         if(e2_try_token(lang, string, E2_TokenKind_Symbol, s("("), &macro_off, 0))
@@ -2012,7 +2046,7 @@ e2_parse_from_string(Arena *arena, E2_ParseState *state, B32 identifier_is_type,
       }
       
       // rjf: first try to resolve as an operator name
-      if(!identifier_mapped)
+      if(qualifiers.node_count == 0 && !identifier_mapped)
       {
         // rjf: string -> op kind
         E2_ExprKind expr_kind = E2_ExprKind_Null;
@@ -2055,7 +2089,7 @@ e2_parse_from_string(Arena *arena, E2_ParseState *state, B32 identifier_is_type,
       }
       
       // rjf: try to resolve it as a macro argument
-      if(!identifier_mapped && state->top_task->expr_kind == E2_ExprKind_Macro)
+      if(qualifiers.node_count == 0 && !identifier_mapped && state->top_task->expr_kind == E2_ExprKind_Macro)
       {
         U64 macro_arg_num = 0;
         {
@@ -2086,6 +2120,7 @@ e2_parse_from_string(Arena *arena, E2_ParseState *state, B32 identifier_is_type,
           done = 1;
           parse.status = E2_ParseStatus_CheckIdentifierIsType;
           parse.identifier = identifier;
+          parse.qualifiers = qualifiers;
           off = start_off;
         }
         else
@@ -2093,6 +2128,7 @@ e2_parse_from_string(Arena *arena, E2_ParseState *state, B32 identifier_is_type,
           identifier_mapped = 1;
           expr = e2_expr(arena, identifier_is_type ? E2_ExprKind_TypeIdentifier : E2_ExprKind_Identifier);
           expr->string = str8_substr(string, token.range);
+          expr->qualifiers = qualifiers;
           state->caller_request_count = 0;
         }
       }
@@ -2651,6 +2687,7 @@ e2_compile_from_expr(Arena *arena, E2_CompileState *state, E2_IRNode *resolve_re
                 done = 1;
                 compile.status = E2_CompileStatus_MissedIdentifierResolution;
                 compile.identifier = e->string;
+                compile.qualifiers = e->qualifiers;
               }
             }break;
             

src/eval2/eval2.h

@@ -326,6 +326,7 @@ struct E2_Expr
   U32 macro_arg_num;
   Rng1U64 src_range;
   String8 string;
+  String8List qualifiers;
 };
 
 typedef struct E2_ExprMapNode E2_ExprMapNode;
@@ -400,6 +401,7 @@ struct E2_Parse
 {
   E2_ParseStatus status;
   String8 identifier;
+  String8List qualifiers;
   E2_Expr *expr;
   String8 member_name;
   E2_Expr *params_expr;
@@ -469,6 +471,7 @@ struct E2_Compile
 {
   E2_CompileStatus status;
   String8 identifier;
+  String8List qualifiers;
   E2_IRNode *irtree;
   String8 member_name;
   E2_IRNode *params_irtree;

src/eval2/tests/eval2_tests.c

@@ -0,0 +1,191 @@
+// Copyright (c) Epic Games Tools
+// Licensed under the MIT license (https://opensource.org/license/mit/)
+
+Test(eval2_regressions)
+{
+  E2_ConsTypeMap *cons_types = e2_cons_type_map_alloc();
+  e2_select_cons_type_map(cons_types);
+  String8 strings[] =
+  {
+    s("int32(*)(int32, int32)"),
+    s("int32 *(*)(int32, int32)"),
+    s("int32 **(*)(int32, int32)"),
+    s("123(1, 2, 3)"),
+    s("int32 (*) [100]"),
+    s("(3 * 4) + 2"),
+    s("cast (float32) 123"),
+    s("int32 *"),
+    s("3 * 4 + 2"),
+    s("int32[100]"),
+    s("3 * 4"),
+    s("foo"),
+    s("foo.bar"),
+    s("[123]"),
+    s("123[456]"),
+    s("2 + (3 * 4)"),
+    s("int32 == int32"),
+    s("123, 456"),
+    s("222.f"),
+    s("123 as float32"),
+    s("cast float32 123"),
+    s("(int32 *)123"),
+    s("bar(a, b) = a + b"),
+    s("foo = 123"),
+    s("1 > 2"),
+    s("1 ? 123 : 456"),
+    s("0 ? 123 : 456"),
+    s("1 ? \"Test\" : 888"),
+    s("'a'"),
+    s("'b'"),
+    s("(float32)222"),
+    s("(float64)222"),
+    s("222.0"),
+    s("123"),
+    s("123 + 456"),
+    s("!1"),
+    s("!0"),
+    s("1 + 1 + 1"),
+    s("40 / 0"),
+    s("123 | 456"),
+    s("123 + "),
+    s("-123"),
+    s("sizeof 123"),
+  };
+  for EachElement(idx, strings)
+  {
+    Temp scratch = scratch_begin(0, 0);
+    String8List msgs = {0};
+    
+    // rjf: string -> expr
+    E2_Expr *expr = &e2_expr_nil;
+    {
+      E2_ParseState state = {0};
+      B32 identifier_is_type = 0;
+      for(;;)
+      {
+        E2_Parse parse = e2_parse_from_string(scratch.arena, &state, identifier_is_type, E2_LangKind_CLike, strings[idx]);
+        identifier_is_type = 0;
+        expr = parse.expr;
+        for EachNode(n, E2_Msg, parse.msgs.first)
+        {
+          str8_list_push(scratch.arena, &msgs, n->string);
+        }
+        if(parse.status == E2_ParseStatus_CheckIdentifierIsType)
+        {
+          identifier_is_type = identifier_is_type || str8_match(parse.identifier, s("int32"), 0);
+          identifier_is_type = identifier_is_type || str8_match(parse.identifier, s("float32"), 0);
+          identifier_is_type = identifier_is_type || str8_match(parse.identifier, s("float64"), 0);
+        }
+        if(e2_parse_status_is_terminal(parse.status))
+        {
+          break;
+        }
+      }
+    }
+    
+    // rjf: expr -> ir tree
+    E2_IRNode *irtree = &e2_irnode_nil;
+    {
+      E2_IRNode *resolve_result = &e2_irnode_nil;
+      E2_Val compile_time_eval_result = {0};
+      E2_CompileState state = {0};
+      for(;;)
+      {
+        E2_Compile compile = e2_compile_from_expr(scratch.arena, &state, resolve_result, compile_time_eval_result, expr);
+        irtree = compile.irtree;
+        resolve_result = &e2_irnode_nil;
+        if(compile.status == E2_CompileStatus_MissedIdentifierResolution)
+        {
+          E2_IRNode *irnode = &e2_irnode_nil;
+          if(str8_match(compile.identifier, s("float32"), 0))
+          {
+            irnode = e2_irnode_type(scratch.arena, e2_type_key_basic(E2_TypeKind_F32));
+          }
+          else if(str8_match(compile.identifier, s("float64"), 0))
+          {
+            irnode = e2_irnode_type(scratch.arena, e2_type_key_basic(E2_TypeKind_F64));
+          }
+          else if(str8_match(compile.identifier, s("int32"), 0))
+          {
+            irnode = e2_irnode_type(scratch.arena, e2_type_key_basic(E2_TypeKind_S32));
+          }
+          else if(str8_match(compile.identifier, s("int64"), 0))
+          {
+            irnode = e2_irnode_type(scratch.arena, e2_type_key_basic(E2_TypeKind_S64));
+          }
+          else
+          {
+            irnode = e2_irnode_const_u64_or_smaller(scratch.arena, 123);
+          }
+          resolve_result = irnode;
+        }
+        if(compile.status == E2_CompileStatus_MemberAccess)
+        {
+          resolve_result = e2_irnode_const_u64_or_smaller(scratch.arena, 456);
+        }
+        if(compile.status == E2_CompileStatus_IndexAccess)
+        {
+          resolve_result = e2_irnode_const_u64_or_smaller(scratch.arena, 111);
+        }
+        if(compile.status == E2_CompileStatus_Call)
+        {
+          resolve_result = e2_irnode_const_u64_or_smaller(scratch.arena, 123456);
+        }
+        if(compile.status == E2_CompileStatus_CompileTimeEval)
+        {
+          String8 bytecode = e2_bytecode_from_irnode(scratch.arena, irtree);
+          E2_InterpState interp_state = {0};
+          E2_SpaceMap space_map = {0};
+          E2_Interp interp = e2_interp_from_bytecode(scratch.arena, &interp_state, &space_map, bytecode);
+          compile_time_eval_result = interp.val;
+        }
+        if(e2_compile_status_is_terminal(compile.status))
+        {
+          break;
+        }
+      }
+    }
+    
+    // rjf: ir tree -> bytecode
+    String8 bytecode = e2_bytecode_from_irnode(scratch.arena, irtree);
+    
+    // rjf: bytecode -> value
+    E2_Val val = {0};
+    {
+      E2_InterpState state = {0};
+      E2_SpaceMap space_map = {0};
+      for(;;)
+      {
+        E2_Interp interp = e2_interp_from_bytecode(scratch.arena, &state, &space_map, bytecode);
+        val = interp.val;
+        if(e2_interp_status_is_terminal(interp.status))
+        {
+          if(interp.status != E2_InterpStatus_Good)
+          {
+            str8_list_pushf(scratch.arena, &msgs, "Interpretation error (%i).", interp.status);
+          }
+          break;
+        }
+      }
+    }
+    
+    // rjf: message string list -> string
+    StringJoin join = {.sep = s(" ")};
+    String8 msgs_string = str8_list_join(scratch.arena, &msgs, &join);
+    
+    // rjf: log
+    String8 log = str8f(scratch.arena, "%S -> %I64d (f32: %f) (f64: %f) *%s* (type: %S) %s%S\n",
+                        strings[idx],
+                        val.s64,
+                        val.f32,
+                        val.f64,
+                        irtree->mode == E2_Mode_Type ? "type" :
+                        irtree->mode == E2_Mode_Value ? "value" :
+                        "address",
+                        e2_string_from_type_key(scratch.arena, irtree->type_key),
+                        msgs_string.size != 0 ? " // " : "", msgs_string);
+    log_info(log);
+    
+    scratch_end(scratch);
+  }
+}

src/raddbg/tests/raddbg_tests.c

@@ -72,7 +72,7 @@ rd_test__exemplar_test_finish(Arena *arena, TestCtx *ctx, String8List test_log_s
     }
     str8_list_push(scratch.arena, &exemplar_data_lines_sanitized, line_trimmed);
   }
-  exemplar_data = str8_list_join(scratch.arena, &exemplar_data_lines_sanitized, &(StringJoin){.sep = s("\n")});
+  exemplar_data = str8_list_join(scratch.arena, &exemplar_data_lines_sanitized, &(StringJoin){.sep = s("\n"), .post = s("\n")});
   
   // rjf: if exemplar data is empty -> just save our output as the new exemplar
   if(exemplar_data.size == 0)

src/scratch/ryan_scratch.c

@@ -43,6 +43,7 @@ entry_point(CmdLine *cmdline)
     // (A)
     // int & B
     // (1 + (int)&B)
+    s("foo!bar"),
     s("int32(*)(int32, int32)"),
     s("int32 *(*)(int32, int32)"),
     s("int32 **(*)(int32, int32)"),

src/text/text.c

@@ -2445,8 +2445,10 @@ txt_token_array_from_data(Arena *arena, TXT_LangKind lang_kind, TXT_TokenPt ctx_
       U64 advance = 1;
       
       //- rjf: adjust escaping state
-      if(active_token_kind != TXT_TokenKind_Null && byte == '\\')
+      B32 escaped_this_step = 0;
+      if(!escaped && active_token_kind != TXT_TokenKind_Null && byte == '\\')
       {
+        escaped_this_step = 1;
         escaped = 1;
       }
       
@@ -2599,7 +2601,12 @@ txt_token_array_from_data(Arena *arena, TXT_LangKind lang_kind, TXT_TokenPt ctx_
           }
         }break;
         case TXT_TokenKind_LineComment:
+        case TXT_TokenKind_Meta:
         {
+          if(byte == '\r' && escaped)
+          {
+            escaped_this_step = 1;
+          }
           if(byte == '\n' && !escaped)
           {
             token_finished = 1;
@@ -2614,14 +2621,6 @@ txt_token_array_from_data(Arena *arena, TXT_LangKind lang_kind, TXT_TokenPt ctx_
             advance = 2;
           }
         }break;
-        case TXT_TokenKind_Meta:
-        {
-          if(byte == '\n' && !escaped)
-          {
-            token_finished = 1;
-            advance = 1;
-          }
-        }break;
       }
       
       //- rjf: finish all tokens if we're at the end of the data
@@ -2654,7 +2653,10 @@ txt_token_array_from_data(Arena *arena, TXT_LangKind lang_kind, TXT_TokenPt ctx_
       }
       
       //- rjf: reset escaped state
-      escaped = 0;
+      if(!escaped_this_step)
+      {
+        escaped = 0;
+      }
       
       //- rjf: advance
       off += advance;
@@ -3734,7 +3736,6 @@ txt_artifact_create(String8 key, B32 *cancel_signal, AC_Status *status_out, U64
       {
         U8 byte = data.str[off];
         U8 next_byte = (off+1 < data.size) ? data.str[off+1] : 0;
-        U8 prev_byte = (off > 0) ? data.str[off-1] : 0;
         B32 off_is_endpoint = 1;
         U64 extra_advance = 0;
         if(byte == '/' && next_byte == '*')
@@ -3757,9 +3758,27 @@ txt_artifact_create(String8 key, B32 *cancel_signal, AC_Status *status_out, U64
         {
           // NOTE(rjf): no-op
         }
-        else if(prev_byte != '\\' && (byte == '"' || byte == '\''))
+        else if(byte == '"' || byte == '\'')
         {
-          // NOTE(rjf): no-op
+          B32 is_escaped = 0;
+          if(off > 0)
+          {
+            for(U64 lookback_off = off-1; lookback_off > 0; lookback_off -= 1)
+            {
+              if(data.str[lookback_off] == '\\')
+              {
+                is_escaped ^= 1;
+              }
+              else
+              {
+                break;
+              }
+            }
+          }
+          if(is_escaped)
+          {
+            off_is_endpoint = 0;
+          }
         }
         else
         {

src/torture/torture.c

@@ -179,7 +179,57 @@ t_run_caller(void *raw_ctx)
       .result_out = &ctx->result,
       .test_out = &test_out,
     };
+    log_scope_begin();
     ctx->test->test_fn(scratch.arena, &test_ctx);
+    LogScopeResult log_scope_result = log_scope_end(scratch.arena);
+    if(log_scope_result.strings[LogMsgKind_Info].size != 0)
+    {
+      String8 current_path = str8f(scratch.arena, "%S/current", g_wdir);
+      String8 exemplar_path = str8f(scratch.arena, "%S/exemplar_%S_%S", g_exemplar_dir, lower_from_str8(scratch.arena, string_from_operating_system(OperatingSystem_CURRENT)), lower_from_str8(scratch.arena, string_from_arch(Arch_CURRENT)));
+      String8 current = log_scope_result.strings[LogMsgKind_Info];
+      String8 exemplar = data_from_file_path(scratch.arena, exemplar_path);
+      write_data_to_file_path(current_path, current);
+      if(exemplar.size == 0)
+      {
+        make_directory(g_exemplar_dir);
+        copy_file_path(exemplar_path, current_path);
+      }
+      else
+      {
+        // TODO(rjf): @hack, see below
+        TestCtx *ctx = &test_ctx;
+        String8List exemplar_lines = str8_split(scratch.arena, exemplar, (U8 *)"\n", 1, StringSplitFlag_KeepEmpties);
+        String8List exemplar_lines_sanitized = {0};
+        for EachNode(n, String8Node, exemplar_lines.first)
+        {
+          String8 line_trimmed = n->string;
+          if(line_trimmed.size != 0 && line_trimmed.str[line_trimmed.size-1] == '\r')
+          {
+            line_trimmed.size -= 1;
+          }
+          str8_list_push(scratch.arena, &exemplar_lines_sanitized, line_trimmed);
+        }
+        StringJoin join = {.sep = s("\n"), .post = s("\n")};
+        String8 exemplar_sanitized = str8_list_join(scratch.arena, &exemplar_lines_sanitized, &join);
+        B32 current_matches_exemplar = str8_match(exemplar_sanitized, current, 0);
+        if(!current_matches_exemplar)
+        {
+          // TODO(rjf): @hack, because we need to do test things in the outer scope, but this is
+          // not a test function - all test helpers should wrap a 100% parameterized layer
+          Arena *arena = scratch.arena;
+          String8 diff_cmd = str8f(scratch.arena, "diff %S %S",
+                                   path_normalized_from_string(scratch.arena, exemplar_path),
+                                   path_normalized_from_string(scratch.arena, current_path));
+          test_outf("Current log does not match exemplar; run `%S`\n", diff_cmd);
+          // TODO(rjf): @hack need to hack this in, because TestCheck assumes `return`
+          ctx->result_out[0] = (TestResult){.fail_file = __FILE__, .fail_line = __LINE__, .fail_cond = "current_matches_exemplar"};
+          if(debugger_is_attached())
+          {
+            Trap();
+          }
+        }
+      }
+    }
   }
   
   if (ctx->result.status == TestStatus_Fail || ctx->result.status == TestStatus_Crash) {
@@ -1054,6 +1104,8 @@ internal void
 t_entry_point(CmdLine *cmdline)
 {
   Temp scratch = scratch_begin(0,0);
+  Log *log = log_alloc();
+  log_select(log);
   U64 exit_code = max_U64;
   
   U64 dashes_size = 9999;

src/torture/torture_main.c

@@ -76,6 +76,7 @@
 #include "stap/stap_parse.h"
 #include "demon/demon_inc.h"
 #include "eval/eval_inc.h"
+#include "eval2/eval2.h"
 #include "dbg_engine/dbg_engine_inc.h"
 #include "eval_visualization/eval_visualization_inc.h"
 #include "font_provider/font_provider_inc.h"
@@ -156,6 +157,7 @@
 #include "stap/stap_parse.c"
 #include "demon/demon_inc.c"
 #include "eval/eval_inc.c"
+#include "eval2/eval2.c"
 #include "dbg_engine/dbg_engine_inc.c"
 #include "eval_visualization/eval_visualization_inc.c"
 #include "font_provider/font_provider_inc.c"
@@ -184,6 +186,7 @@
 #include "rdi_from_dwarf/tests/rdi_from_dwarf_tests.c"
 #include "rdi_from_pdb/tests/rdi_from_pdb_tests.c"
 #include "raddbg/tests/raddbg_tests.c"
+#include "eval2/tests/eval2_tests.c"
 
 internal B32 frame(void) { return 0; }