raddbg exemplar tests -> use specific hash version, name exemplars by their platform, skip if test data unavailable

project.raddbg

@@ -41,6 +41,6 @@ target:
 {
   executable: "build/torture.exe"
   working_directory: build
-  arguments: "raddbg/*"
+  arguments: "raddbg/* --gui"
   enabled: 1
 }

src/base/base_test.h

@@ -23,7 +23,18 @@ struct TestResult
   char *fail_cond;
 };
 
-#define TEST_FUNCTION_SIG(name) void name(Arena *arena, CmdLine *cmdline, String8 test_exemplars_path, String8 test_artifacts_path, TestResult *result_out, String8List *test_out)
+typedef struct TestCtx TestCtx;
+struct TestCtx
+{
+  CmdLine *cmdline;
+  String8 exemplars_path;
+  String8 artifacts_path;
+  String8 input_data_path;
+  TestResult *result_out;
+  String8List *test_out;
+};
+
+#define TEST_FUNCTION_SIG(name) void name(Arena *arena, TestCtx *ctx)
 #define TEST_FUNCTION_DEF(name) TEST_FUNCTION_SIG(test__##name)
 typedef TEST_FUNCTION_SIG(TestFunctionType);
 
@@ -76,17 +87,17 @@ TEST_FUNCTION_DEF(name)
 #define SkippedTest(name) DeclareTest(name, 1)\
 TEST_FUNCTION_DEF(name)
 
-#define test_out(string) str8_list_push(arena, test_out, (string))
-#define test_outf(...) str8_list_pushf(arena, test_out, __VA_ARGS__)
+#define test_out(string) str8_list_push(arena, ctx->test_out, (string))
+#define test_outf(...) str8_list_pushf(arena, ctx->test_out, __VA_ARGS__)
 
 #define TestCheck(c) do { if (!(c)) {\
-*result_out = (TestResult){ .fail_file = __FILE__, .fail_line = __LINE__, .fail_cond = Stringify(c) };\
+ctx->result_out[0] = (TestResult){ .fail_file = __FILE__, .fail_line = __LINE__, .fail_cond = Stringify(c) };\
 if(debugger_is_attached()) { Trap(); }\
 return;\
 } } while(0)
 
 #define TestSkip() do {\
-*result_out = (TestResult){ .status = TestStatus_Skip };\
+ctx->result_out[0] = (TestResult){ .status = TestStatus_Skip };\
 return;\
 } while(0)
 

src/raddbg/tests/raddbg_tests.c

@@ -5,10 +5,10 @@
 //~ rjf: Debugger Testing IPC Driving Helpers
 
 internal String8
-rd_test__mule_main_path(Arena *arena)
+rd_test__test_binary_path(Arena *arena, TestCtx *ctx, String8 name)
 {
-  String8 mule_main_exe_path = str8f(arena, "%S/%S%S", get_process_info()->binary_path, s("mule_main"), program_ext_postfix_from_os(OperatingSystem_CURRENT));
-  return mule_main_exe_path;
+  String8 path = str8f(arena, "%S/%S/%S%S", ctx->input_data_path, name, name, program_ext_postfix_from_os(OperatingSystem_CURRENT));
+  return path;
 }
 
 internal String8
@@ -20,11 +20,11 @@ rd_test__raddbg_path(Arena *arena, CmdLine *cmdline)
 }
 
 internal Process
-rd_test__open_debugger(CmdLine *cmdline, String8 test_artifacts_path, String8 target_cmd_line)
+rd_test__open_debugger(TestCtx *ctx, String8 target_cmd_line)
 {
   Temp scratch = scratch_begin(0, 0);
-  String8 raddbg_path = rd_test__raddbg_path(scratch.arena, cmdline);
-  String8 user_file_path = str8f(scratch.arena, "%S/test.raddbg_user", test_artifacts_path);
+  String8 raddbg_path = rd_test__raddbg_path(scratch.arena, ctx->cmdline);
+  String8 user_file_path = str8f(scratch.arena, "%S/test.raddbg_user", ctx->artifacts_path);
   delete_file_at_path(user_file_path);
   Process process = launch_cmd_linef("%S --gen_crash_dump --no_focus_on_stop --user:%S --logs:logs %S", raddbg_path, user_file_path, target_cmd_line);
   scratch_end(scratch);
@@ -38,11 +38,11 @@ rd_test__close_debugger(Process process)
 }
 
 internal String8
-rd_test__ipc_cmd(Arena *arena, CmdLine *cmdline, String8 test_artifacts_path, U64 debugger_pid, String8 string)
+rd_test__ipc_cmd(Arena *arena, TestCtx *ctx, U64 debugger_pid, String8 string)
 {
   Temp scratch = scratch_begin(&arena, 1);
-  String8 raddbg_path = rd_test__raddbg_path(scratch.arena, cmdline);
-  String8 stdout_path = str8f(scratch.arena, "%S/ipc_output", test_artifacts_path);
+  String8 raddbg_path = rd_test__raddbg_path(scratch.arena, ctx->cmdline);
+  String8 stdout_path = str8f(scratch.arena, "%S/ipc_output", ctx->artifacts_path);
   Process process = launch_cmd_linef("%S --ipc --pid:%I64u %S > %S", raddbg_path, debugger_pid, string, stdout_path);
   process_join(process, max_U64, 0);
   String8 response = data_from_file_path(arena, stdout_path);
@@ -55,18 +55,18 @@ rd_test__ipc_cmd(Arena *arena, CmdLine *cmdline, String8 test_artifacts_path, U6
 //~ rjf: Exemplar Test Helpers
 
 internal void
-rd_test__exemplar_test_finish(Arena *arena, String8List test_log_strings, String8 test_exemplars_path, String8 test_artifacts_path, TestResult *result_out, String8List *test_out)
+rd_test__exemplar_test_finish(Arena *arena, TestCtx *ctx, String8List test_log_strings)
 {
   Temp scratch = scratch_begin(&arena, 1);
   
   // rjf: combine all current test log output; write to 'current' data
   String8 test_log_current = str8_list_join(scratch.arena, &test_log_strings, 0);
-  String8 test_log_current_path = str8f(scratch.arena, "%S/current", test_artifacts_path);
+  String8 test_log_current_path = str8f(scratch.arena, "%S/current", ctx->artifacts_path);
   write_data_to_file_path(test_log_current_path, test_log_current);
   
   // rjf: load test log exemplar
-  make_directory(test_exemplars_path);
-  String8 exemplar_path = str8f(scratch.arena, "%S/exemplar", test_exemplars_path);
+  make_directory(ctx->exemplars_path);
+  String8 exemplar_path = str8f(scratch.arena, "%S/exemplar_%S", ctx->exemplars_path, lower_from_str8(scratch.arena, string_from_operating_system(OperatingSystem_CURRENT)));
   String8 exemplar_data = data_from_file_path(scratch.arena, exemplar_path);
   
   // rjf: if exemplar data is empty -> just save our output as the new exemplar
@@ -96,13 +96,26 @@ rd_test__exemplar_test_finish(Arena *arena, String8List test_log_strings, String
 //~ rjf: Test Helpers
 
 internal void
-rd_test__stepping_regressions(Arena *arena, CmdLine *cmdline, String8 test_exemplars_path, String8 test_artifacts_path, TestResult *result_out, String8List *test_out, String8 target_cmdline, String8 start_symbol, B32 step_over_on_even)
+rd_test__stepping_regressions(Arena *arena, TestCtx *ctx, String8 target_binary, String8 start_symbol, B32 step_over_on_even)
 {
   Temp scratch = scratch_begin(&arena, 1);
   String8List test_log_strings = {0};
   
+  // rjf: get binary path
+  String8 binary_path = rd_test__test_binary_path(arena, ctx, target_binary);
+  B32 binary_exists_locally = (properties_from_file_path(binary_path).modified != 0);
+  
+  // rjf: if binary does not exist -> skip this test, 
+  if(!binary_exists_locally)
+  {
+    TestSkip();
+  }
+  
+  // rjf: otherwise run
+  else
+  {
     // rjf: start debugger
-  Process debugger = rd_test__open_debugger(cmdline, test_artifacts_path, target_cmdline);
+    Process debugger = rd_test__open_debugger(ctx, binary_path);
     U64 debugger_pid = pid_from_process(debugger);
     
     // rjf: step & gather info
@@ -132,13 +145,13 @@ rd_test__stepping_regressions(Arena *arena, CmdLine *cmdline, String8 test_exemp
             step_cmd = next_step_cmd;
             MemoryZeroStruct(&next_step_cmd);
           }
-        String8 step_response = rd_test__ipc_cmd(scratch2.arena, cmdline, test_artifacts_path, debugger_pid, step_cmd);
+          String8 step_response = rd_test__ipc_cmd(scratch2.arena, ctx, debugger_pid, step_cmd);
           (void)step_response;
           cmd_idx += 1;
         }
         
         // rjf: query state, parse
-      String8 state_response = rd_test__ipc_cmd(scratch2.arena, cmdline, test_artifacts_path, debugger_pid, s("state"));
+        String8 state_response = rd_test__ipc_cmd(scratch2.arena, ctx, debugger_pid, s("state"));
         MD_Node *state_response_tree = md_tree_from_string(scratch2.arena, state_response);
         MD_Node *state = md_child_from_string(state_response_tree, s("state"), 0);
         
@@ -187,42 +200,56 @@ rd_test__stepping_regressions(Arena *arena, CmdLine *cmdline, String8 test_exemp
     }
     
     // rjf: end exemplar test
-  rd_test__exemplar_test_finish(arena, test_log_strings, test_exemplars_path, test_artifacts_path, result_out, test_out);
+    rd_test__exemplar_test_finish(arena, ctx, test_log_strings);
     
     // rjf: close debugger
     rd_test__close_debugger(debugger);
+  }
   
   scratch_end(scratch);
 }
 
 internal void
-rd_test__eval_regressions(Arena *arena, CmdLine *cmdline, String8 test_exemplars_path, String8 test_artifacts_path, TestResult *result_out, String8List *test_out, String8 target_cmdline, String8 target_line)
+rd_test__eval_regressions(Arena *arena, TestCtx *ctx, String8 target_binary, String8 target_line)
 {
   Temp scratch = scratch_begin(&arena, 1);
   String8List test_log_strings = {0};
   
+  // rjf: get binary path
+  String8 binary_path = rd_test__test_binary_path(arena, ctx, target_binary);
+  B32 binary_exists_locally = (properties_from_file_path(binary_path).modified != 0);
+  
+  // rjf: if binary does not exist -> skip this test, 
+  if(!binary_exists_locally)
+  {
+    TestSkip();
+  }
+  
+  // rjf: otherwise run
+  else
+  {
     // rjf: start debugger
-  Process debugger = rd_test__open_debugger(cmdline, test_artifacts_path, target_cmdline);
+    Process debugger = rd_test__open_debugger(ctx, binary_path);
     U64 debugger_pid = pid_from_process(debugger);
     
     // rjf: query state
     U64 initial_stop_count = 0;
     {
-    String8 response = rd_test__ipc_cmd(scratch.arena, cmdline, test_artifacts_path, debugger_pid, s("state"));
+      String8 response = rd_test__ipc_cmd(scratch.arena, ctx, debugger_pid, s("state"));
       MD_Node *state = md_child_from_string(md_tree_from_string(scratch.arena, response), s("state"), 0);
       MD_Node *stop_count = md_child_from_string(state, s("stop_count"), 0);
       try_u64_from_str8_c_rules(stop_count->first->string, &initial_stop_count);
     }
     
     // rjf: run to target line
-  rd_test__ipc_cmd(scratch.arena, cmdline, test_artifacts_path, debugger_pid, str8f(scratch.arena, "run_to_line %S", target_line));
+    rd_test__ipc_cmd(scratch.arena, ctx, debugger_pid, str8f(scratch.arena, "run_to_line %S", target_line));
     
     // rjf: wait for stop, collect locals
     String8List locals = {0};
     for(;;)
     {
       Temp scratch2 = scratch_begin(&scratch.arena, 1);
-    String8 response = rd_test__ipc_cmd(scratch.arena, cmdline, test_artifacts_path, debugger_pid, s("state"));
+      String8 response = rd_test__ipc_cmd(scratch.arena, ctx, debugger_pid, s("state"));
       MD_Node *state = md_child_from_string(md_tree_from_string(scratch.arena, response), s("state"), 0);
       MD_Node *stop_count = md_child_from_string(state, s("stop_count"), 0);
       U64 new_stop_count = 0;
@@ -247,7 +274,7 @@ rd_test__eval_regressions(Arena *arena, CmdLine *cmdline, String8 test_exemplars
     }
     StringJoin join = {.sep = s(" ; ")};
     String8 eval_msg = str8_list_join(scratch.arena, &eval_cmd_strings, &join);
-  String8 response = rd_test__ipc_cmd(scratch.arena, cmdline, test_artifacts_path, debugger_pid, eval_msg);
+    String8 response = rd_test__ipc_cmd(scratch.arena, ctx, debugger_pid, eval_msg);
     MD_Node *response_tree = md_tree_from_string(scratch.arena, response);
     for MD_EachNode(n, response_tree->first)
     {
@@ -259,10 +286,11 @@ rd_test__eval_regressions(Arena *arena, CmdLine *cmdline, String8 test_exemplars
     }
     
     // rjf: end exemplar test
-  rd_test__exemplar_test_finish(arena, test_log_strings, test_exemplars_path, test_artifacts_path, result_out, test_out);
+    rd_test__exemplar_test_finish(arena, ctx, test_log_strings);
     
     // rjf: close debugger
     rd_test__close_debugger(debugger);
+  }
   
   scratch_end(scratch);
 }
@@ -270,29 +298,26 @@ rd_test__eval_regressions(Arena *arena, CmdLine *cmdline, String8 test_exemplars
 ////////////////////////////////
 //~ rjf: Tests
 
-#if 0
-Test(mule_main_step_regressions_0)
+Test(mule_main_9ff1e58f_step_regressions_0)
 {
-  String8 exe_path = {0};
-  rd_test__stepping_regressions(arena, cmdline, test_exemplars_path, test_artifacts_path, result_out, test_out, rd_test__mule_main_path(arena), s("mule_main"), 0);
+  rd_test__stepping_regressions(arena, ctx, s("mule_main_9ff1e58f"), s("mule_main"), 0);
 }
 
-Test(mule_main_step_regressions_1)
+Test(mule_main_9ff1e58f_step_regressions_1)
 {
-  rd_test__stepping_regressions(arena, cmdline, test_exemplars_path, test_artifacts_path, result_out, test_out, rd_test__mule_main_path(arena), s("mule_main"), 1);
+  rd_test__stepping_regressions(arena, ctx, s("mule_main_9ff1e58f"), s("mule_main"), 1);
 }
 
-Test(control_flow_step_regressions)
+Test(mule_main_9ff1e58f_control_flow_step_regressions)
 {
-  rd_test__stepping_regressions(arena, cmdline, test_exemplars_path, test_artifacts_path, result_out, test_out, rd_test__mule_main_path(arena), s("mule_main"), 0);
+  rd_test__stepping_regressions(arena, ctx, s("mule_main_9ff1e58f"), s("control_flow_stepping_tests"), 0);
 }
 
-SkippedTest(type_coverage_eval_regressions)
+SkippedTest(mule_main_9ff1e58f_type_coverage_eval_regressions)
 {
   // TODO(rjf): see @eval_regressions
-  rd_test__eval_regressions(arena, cmdline, test_exemplars_path, test_artifacts_path, result_out, test_out, rd_test__mule_main_path(arena), s("C:/devel/raddebugger/src/mule/mule_main.cpp:723"));
+  rd_test__eval_regressions(arena, ctx, s("mule_main_9ff1e58f"), s("C:/devel/raddebugger/src/mule/mule_main.cpp:723"));
 }
-#endif
 
 #if 0
 ////////////////////////////////

src/torture/torture.c

@@ -5,6 +5,7 @@
 global String8      g_stdout_file_name = str8_lit_comp("torture.out");
 global String8      g_wdir;
 global String8      g_exemplar_dir;
+global String8      g_input_data_dir;
 global String8      g_out = str8_lit_comp("torture_artifacts");
 global B32          g_verbose;
 global B32          g_redirect_stdout = 1;
@@ -169,7 +170,16 @@ t_run_caller(void *raw_ctx)
   if (ctx->test->skip) {
     ctx->result.status = TestStatus_Skip;
   } else {
-    ctx->test->test_fn(scratch.arena, ctx->cmdline, g_exemplar_dir, g_wdir, &ctx->result, &test_out);
+    TestCtx test_ctx =
+    {
+      .cmdline = ctx->cmdline,
+      .exemplars_path = g_exemplar_dir,
+      .artifacts_path = g_wdir,
+      .input_data_path = g_input_data_dir,
+      .result_out = &ctx->result,
+      .test_out = &test_out,
+    };
+    ctx->test->test_fn(scratch.arena, &test_ctx);
   }
   
   if (ctx->result.status == TestStatus_Fail || ctx->result.status == TestStatus_Crash) {
@@ -1281,6 +1291,13 @@ t_entry_point(CmdLine *cmdline)
         g_exemplar_dir = str8f(scratch.arena, "%S/data/test_exemplars/%S", root_dir_path, test->label);
       }
       
+      // rjf: find input data directory
+      {
+        String8 binary_dir_path = get_process_info()->binary_path;
+        String8 root_dir_path = str8_chop_last_slash(binary_dir_path);
+        g_input_data_dir = str8f(scratch.arena, "%S/local/test_data", root_dir_path);
+      }
+      
       // setup output directory
       g_wdir = str8f(scratch.arena, "%S/%S", g_out, test->label);
       g_wdir = full_path_from_path(scratch.arena, g_wdir);

src/torture/torture.h

@@ -24,7 +24,7 @@ typedef struct
 #define T_Ok(c) TestCheck(c)
 
 #define T_MatchLinef(out, ...) T_Ok(t_match_linef(out, __VA_ARGS__))
-#define t_outf(...) str8_list_pushf(arena, test_out, ## __VA_ARGS__)
+#define t_outf(...) str8_list_pushf(arena, ctx->test_out, ## __VA_ARGS__)
 
 ////////////////////////////////////////////////////////////////