update torture

* factor test registration logic out of the macro to improve debuggability
* expand the test registration logic
* rename group -> layer for consistency
* fix indentation in the slow tests summary

src/base/base_test.c

@@ -1,2 +1,22 @@
 // Copyright (c) Epic Games Tools
 // Licensed under the MIT license (https://opensource.org/license/mit/)
+
+#if BUILD_TESTS
+
+internal void
+base_register_test(char *func_name_cstr, TestFunctionType *fn, char *file_path_cstr, int line, int skip)
+{
+  String8 file_path       = str8_cstring(file_path_cstr);
+  U64     src_min         = str8_find_needle(file_path, 0, s("src/"), StringMatchFlag_SlashInsensitive);
+  U64     src_max         = src_min + str8_lit("src/").size;
+  U64     layer_slash_max = str8_find_needle(file_path, src_max, s("/"), StringMatchFlag_SlashInsensitive);
+
+  TestInfo *t = &test_infos[test_infos_count++];
+  t->layer     = str8_substr(file_path, r1u64(src_max, layer_slash_max));
+  t->label     = str8_cstring(func_name_cstr);
+  t->decl_line = line;
+  t->skip      = skip;
+  t->test_fn   = fn;
+}
+
+#endif

src/base/base_test.h

@@ -49,31 +49,25 @@ struct TestInfo
 };
 
 #if BUILD_TESTS
-global U64 test_infos_count = 0;
+
+// alloc storage for the tests
+global U16      test_infos_count    = 0;
 global TestInfo test_infos[0xffff] = {0};
-#define AddTest(name, file_name, line, skip_, ...) \
+
-TEST_FUNCTION_DEF(name);\
+internal void base_register_test(char *func_name, TestFunctionType *fn, char *file_path, int line, int skip);
-__VA_ARGS__ void add_test__##name(void)\
+
-{\
+#define AddTest(name, file_path, line, skip_, ...) \
-String8 file = str8_lit(file_name);\
+  TEST_FUNCTION_DEF(name);                         \
-U64 src_pos = str8_find_needle(file, 0, s("src/"), StringMatchFlag_SlashInsensitive);\
+  __VA_ARGS__ void add_test__##name(void) { base_register_test(Stringify(name), test__##name, file_path, line, skip_); }
-String8 layer_folder = str8_skip(file, src_pos+4);\
+
-U64 layer_slash_pos = str8_find_needle(layer_folder, 0, s("/"), StringMatchFlag_SlashInsensitive);\
-String8 layer_name = str8_prefix(layer_folder, layer_slash_pos);\
-test_infos[test_infos_count].layer     = layer_name;\
-test_infos[test_infos_count].label     = str8_lit(#name);\
-test_infos[test_infos_count].decl_line = (line);\
-test_infos[test_infos_count].skip      = (skip_);\
-test_infos[test_infos_count].test_fn   = test__##name;\
-test_infos_count += 1;\
-}
 # if COMPILER_MSVC
 #  pragma section(".CRT$XCU", read)
 #  define DeclareTest(name, skip)                                                                                      \
-AddTest(name, __FILE__, __LINE__, (skip))\
+  /* register test          */ AddTest(name, __FILE__, __LINE__, (skip))                                               \
-__declspec(allocate(".CRT$XCU")) void (*add_test_ptr__##name)(void) = add_test__##name;\
+  /* alloc function pointer */ __declspec(allocate(".CRT$XCU")) void (*add_test_ptr__##name)(void) = add_test__##name; \
-__pragma(comment(linker, "/include:" Stringify(add_test_ptr__##name)))
+  /* do not GC test caller  */ __pragma(comment(linker, "/include:" Stringify(add_test_ptr__##name)))
 # elif COMPILER_GCC || COMPILER_CLANG
+// clang and gcc allocate memory for the function pointer automatically
 #  define DeclareTest(name, skip) AddTest(name, __FILE__, __LINE__, (skip), __attribute__((constructor)))
 # else
 #  error DeclareTest not defined for this compiler.
@@ -82,23 +76,22 @@ __pragma(comment(linker, "/include:" Stringify(add_test_ptr__##name)))
 # define DeclareTest(name, skip)
 #endif
 
-#define Test(name) DeclareTest(name, 0)\
+#define Test(name)        DeclareTest(name, 0) TEST_FUNCTION_DEF(name)
-TEST_FUNCTION_DEF(name)
+#define SkippedTest(name) DeclareTest(name, 1) TEST_FUNCTION_DEF(name)
-#define SkippedTest(name) DeclareTest(name, 1)\
-TEST_FUNCTION_DEF(name)
 
+#define TestCheck(c) do { if (!(c)) {                                                                                                          \
+  /* record failed check     */ ctx->result_out[0] = (TestResult){ .fail_file = __FILE__, .fail_line = __LINE__, .fail_cond = Stringify(c) }; \
+  /* under debugger? -> trap */ if(debugger_is_attached()) { Trap(); }                                                                        \
+  /* exit test               */ return;                                                                                                       \
+} } while(0)
+
+#define TestSkip() do {                                           \
+  ctx->result_out[0] = (TestResult){ .status = TestStatus_Skip }; \
+  return;                                                         \
+} while(0)
+
+// test log
 #define test_out(string) str8_list_push(arena, ctx->test_out, (string))
 #define test_outf(...)   str8_list_pushf(arena, ctx->test_out, __VA_ARGS__)
 
-#define TestCheck(c) do { if (!(c)) {\
-ctx->result_out[0] = (TestResult){ .fail_file = __FILE__, .fail_line = __LINE__, .fail_cond = Stringify(c) };\
-if(debugger_is_attached()) { Trap(); }\
-return;\
-} } while(0)
-
-#define TestSkip() do {\
-ctx->result_out[0] = (TestResult){ .status = TestStatus_Skip };\
-return;\
-} while(0)
-
 #endif // BASE_TEST_H

src/torture/torture.c

@@ -40,7 +40,7 @@ t_name_from_test_info(Arena *arena, TestInfo *test_info)
 }
 
 internal String8List
-t_test_group_from_name(Arena *arena, String8 pattern)
+t_test_layer_from_name(Arena *arena, String8 pattern)
 {
   Temp scratch = scratch_begin(&arena, 1);
   String8List matches = {0};
@@ -120,6 +120,7 @@ t_delete_dir(String8 path)
     
     if (info.props.flags & FilePropertyFlag_IsFolder) {
       t_delete_dir(str8f(scratch.arena, "%S/%S", path, info.name));
+
       continue;
     }
     
@@ -1030,7 +1031,7 @@ t_help(void)
   fprintf(stderr, " Usage: torture [Options] <Input>\n\n");
   fprintf(stderr, " Options:\n");
   fprintf(stderr, "   -list                 Print available test targets\n");
-  fprintf(stderr, "   --gui                 Launch debugger with window\n");
+  fprintf(stderr, "   -gui                  Launch debugger with window\n");
   fprintf(stderr, "   -cl:<path>            Override default cl path\n");
   fprintf(stderr, "   -clang:<path>         Override default clang path\n");
   fprintf(stderr, "   -gcc:<path>           Override default gcc path\n");
@@ -1155,17 +1156,12 @@ t_entry_point(CmdLine *cmdline)
         String8 test_name = t_name_from_test_info(scratch.arena, test_info);
         
         if (str8_match_wildcard(test_name, t, StringMatchFlag_CaseInsensitive)) {
-          // TODO(rjf): do we really need to mutate the test infos here? this test won't
-          // even run, so I am not sure what setting the bit does - we already collect
-          // the tests that we *will* run
-#if 0
           // set skip flag
           switch (mode) {
             case Mode_Default: break;
-            case Mode_Skip:  g_torture_tests[test_idx]->skip = 1; break;
+            case Mode_Skip:  test_info->skip = 1; break;
-            case Mode_Force: g_torture_tests[test_idx]->skip = 0; break;
+            case Mode_Force: test_info->skip = 0; break;
           }
-#endif
           
           // append test when not in skipping mode
           if ( ! hash_map_search_string_u64(&hm, test_name)) {
@@ -1238,12 +1234,12 @@ t_entry_point(CmdLine *cmdline)
     U64Array target_indices = u64_array_from_list(scratch.arena, &targets);
     
     U64 max_label_size = 0;
-    U64 max_group_size = 0;
+    U64 max_layer_size = 0;
     for EachIndex(i, target_indices.count) {
       U64 test_idx = target_indices.v[i];
       TestInfo *test_info = g_sorted_test_infos[test_idx];
       max_label_size = Max(max_label_size, test_info->label.size);
-      max_group_size = Max(max_group_size, test_info->layer.size);
+      max_layer_size = Max(max_layer_size, test_info->layer.size);
     }
     
     U64 run_counters[TestStatus_COUNT] = {0};
@@ -1268,7 +1264,7 @@ t_entry_point(CmdLine *cmdline)
       U64 curr_digit_count = count_digits_u64(i+1, 10);
       int idx_align_space_count = (int)(max_digit_count - curr_digit_count);
       fprintf(stdout, "[%.*s%llu/%llu] ", idx_align_space_count, spaces, (unsigned long long)i+1, (unsigned long long)target_indices.count);
-      fprintf(stdout, "%.*s %.*s/ %.*s", str8_varg(test->layer), (int)(max_group_size - test->layer.size), spaces, str8_varg(test->label));
+      fprintf(stdout, "%.*s %.*s/ %.*s", str8_varg(test->layer), (int)(max_layer_size - test->layer.size), spaces, str8_varg(test->label));
       fprintf(stdout, " %.*s ", (int)dots_count, dots);
       fflush(stdout);
       
@@ -1375,6 +1371,7 @@ t_entry_point(CmdLine *cmdline)
       fprintf(stderr, "  Skipped  %llu\n", (unsigned long long)run_counters[TestStatus_Skip]);
       fprintf(stderr, "  Time     %.*s\n", str8_varg(total_time_str));
       
+      // count slow tests
       U64 slow_count = 0;
       for EachElement(i, slowest) {
         Slowest s = slowest[i];
@@ -1382,26 +1379,28 @@ t_entry_point(CmdLine *cmdline)
         slow_count += 1;
       }
       
-      if (slow_count > 3) {
+      if (slow_count > 1) {
         U64 label_max = 0;
-        U64 group_max = 0;
+        U64 layer_max = 0;
         for EachElement(i, slowest) {
           Slowest s = slowest[i];
           label_max = Max(g_sorted_test_infos[s.target_idx]->label.size, label_max);
-          group_max = Max(g_sorted_test_infos[s.target_idx]->layer.size, group_max);
+          layer_max = Max(g_sorted_test_infos[s.target_idx]->layer.size, layer_max);
         }
         
         fprintf(stderr, "  \nSlow Tests\n");
         for EachElement(i, slowest) {
           Slowest s = slowest[i];
           if (s.target_idx >= test_infos_count) { break; }
-          TestInfo *test_info = g_sorted_test_infos[i];
+
+          TestInfo *test_info    = g_sorted_test_infos[s.target_idx];
           String8   elapsed_time = string_from_elapsed_time(scratch.arena, date_time_from_micro_seconds(s.d));
+
           fprintf(stderr, "    %.*s %.*s/ %.*s %.*s %.*s\n",
                   str8_varg(test_info->layer),
-                  (int)(group_max - test_info->layer.size), spaces,
+                  (int)(layer_max - test_info->layer.size), spaces,
                   str8_varg(test_info->label),
-                  (int)(label_max - test_info->layer.size) + 4, dots,
+                  (int)(label_max - test_info->label.size) + 4, dots,
                   str8_varg(elapsed_time));
         }
       }

src/torture/torture.h

@@ -5,12 +5,6 @@
 
 ////////////////////////////////
 
-#define T_RESET  "\x1b[0m"
-#define T_RED    "\x1b[31m"
-#define T_GREEN  "\x1b[32m"
-#define T_YELLOW "\x1b[33m"
-#define T_BLUE   "\x1b[34m"
-
 typedef struct
 {
   TestInfo *test;
@@ -19,14 +13,7 @@ typedef struct
   TestResult result;
 } T_RunCtx;
 
-#define TEST(name) Test(name)
+////////////////////////////////
-#define SKIP(name) SkippedTest(name)
-#define T_Ok(c) TestCheck(c)
-
-#define T_MatchLinef(out, ...) T_Ok(t_match_linef(out, __VA_ARGS__))
-#define t_outf(...) str8_list_pushf(arena, ctx->test_out, ## __VA_ARGS__)
-
-////////////////////////////////////////////////////////////////
 
 #define TIMEOUT_US(x)  (x)
 #define TIMEOUT_MS(x)  TIMEOUT_US((x)*1000ull)
@@ -36,9 +23,17 @@ typedef struct
 #define ENDT_MS(x)  TIMEOUT_US((x)*1000ull)
 #define ENDT_SEC(x) TIMEOUT_MS((x)*1000ull)
 
-////////////////////////////////////////////////////////////////
+////////////////////////////////
 
-// output directory
+#define T_RESET  "\x1b[0m"
+#define T_RED    "\x1b[31m"
+#define T_GREEN  "\x1b[32m"
+#define T_YELLOW "\x1b[33m"
+#define T_BLUE   "\x1b[34m"
+
+////////////////////////////////
+
+// artifacts directory helpers
 internal B32     t_write_file_list(String8 name, String8List data);
 internal B32     t_write_file(String8 name, String8 data);
 internal String8 t_read_file(Arena *arena, String8 name);
@@ -59,6 +54,7 @@ internal String8 t_radlink_path(void);
 internal String8 t_cwd_path(void);
 internal String8 t_src_path(void);
 
+// process helpers
 internal B32 t_invoke(String8 exe, String8 cmdline, U64 timeout);
 internal B32 t_invoke_cl(char *fmt, ...);
 internal B32 t_invoke_linkerf(char *fmt, ...);
@@ -68,16 +64,18 @@ internal void t_kill_all(String8 pattern);
 #define t_invoke_linker_timeoutf(t, f, ...) t_invoke_linker_timeout(push_str8f(arena, f, ##__VA_ARGS__), t)
 #define t_invoke_linker(c)                  t_invoke_linker_timeout(c, max_U64)
 
-internal void t_kill_all(String8 pattern);
-
 internal String8 t_chop_line(String8 *string);
 internal B32     t_match_line(String8 *output, String8 expected_line);
 internal B32     t_match_linef(String8 *output, char *fmt, ...);
+#define T_MatchLinef(out, ...) T_Ok(t_match_linef(out, __VA_ARGS__))
 
 // files helper
 internal String8List t_file_paths_from_dir(Arena *arena, String8 dir);
 
-// printer
+// TODO: deprecate
 internal void t_infof(char *fmt, ...);
 internal void t_errorf(char *fmt, ...);
+#define TEST(name) Test(name)
+#define SKIP(name) SkippedTest(name)
+#define T_Ok(c)    TestCheck(c)