From 8ef58def99ef69807e9b9e80447b2a67cf33c9ad Mon Sep 17 00:00:00 2001 From: Nikita Smith Date: Thu, 18 Jun 2026 22:39:37 -0700 Subject: [PATCH] update torture * factor test registration logic out of the macro to improve debuggability * expand the test registration logic * rename group -> layer for consistency * fix indentation in the slow tests summary --- src/base/base_test.c | 20 ++++++++++ src/base/base_test.h | 89 ++++++++++++++++++++----------------------- src/torture/torture.c | 39 +++++++++---------- src/torture/torture.h | 44 ++++++++++----------- 4 files changed, 101 insertions(+), 91 deletions(-) diff --git a/src/base/base_test.c b/src/base/base_test.c index ae447aab..5a2e302f 100644 --- a/src/base/base_test.c +++ b/src/base/base_test.c @@ -1,2 +1,22 @@ // Copyright (c) Epic Games Tools // Licensed under the MIT license (https://opensource.org/license/mit/) + +#if BUILD_TESTS + +internal void +base_register_test(char *func_name_cstr, TestFunctionType *fn, char *file_path_cstr, int line, int skip) +{ + String8 file_path = str8_cstring(file_path_cstr); + U64 src_min = str8_find_needle(file_path, 0, s("src/"), StringMatchFlag_SlashInsensitive); + U64 src_max = src_min + str8_lit("src/").size; + U64 layer_slash_max = str8_find_needle(file_path, src_max, s("/"), StringMatchFlag_SlashInsensitive); + + TestInfo *t = &test_infos[test_infos_count++]; + t->layer = str8_substr(file_path, r1u64(src_max, layer_slash_max)); + t->label = str8_cstring(func_name_cstr); + t->decl_line = line; + t->skip = skip; + t->test_fn = fn; +} + +#endif diff --git a/src/base/base_test.h b/src/base/base_test.h index 7d4f3c9a..f17c8f1c 100644 --- a/src/base/base_test.h +++ b/src/base/base_test.h @@ -17,20 +17,20 @@ TestStatus; typedef struct TestResult TestResult; struct TestResult { - TestStatus status; - char *fail_file; - int fail_line; - char *fail_cond; + TestStatus status; + char *fail_file; + int fail_line; + char *fail_cond; }; typedef struct TestCtx TestCtx; struct TestCtx { - CmdLine *cmdline; - String8 exemplars_path; - String8 artifacts_path; - String8 input_data_path; - TestResult *result_out; + CmdLine *cmdline; + String8 exemplars_path; + String8 artifacts_path; + String8 input_data_path; + TestResult *result_out; String8List *test_out; }; @@ -41,39 +41,33 @@ typedef TEST_FUNCTION_SIG(TestFunctionType); typedef struct TestInfo TestInfo; struct TestInfo { - String8 layer; - String8 label; - S64 decl_line; - B32 skip; + String8 layer; + String8 label; + S64 decl_line; + B32 skip; TestFunctionType *test_fn; }; #if BUILD_TESTS -global U64 test_infos_count = 0; + +// alloc storage for the tests +global U16 test_infos_count = 0; global TestInfo test_infos[0xffff] = {0}; -#define AddTest(name, file_name, line, skip_, ...) \ -TEST_FUNCTION_DEF(name);\ -__VA_ARGS__ void add_test__##name(void)\ -{\ -String8 file = str8_lit(file_name);\ -U64 src_pos = str8_find_needle(file, 0, s("src/"), StringMatchFlag_SlashInsensitive);\ -String8 layer_folder = str8_skip(file, src_pos+4);\ -U64 layer_slash_pos = str8_find_needle(layer_folder, 0, s("/"), StringMatchFlag_SlashInsensitive);\ -String8 layer_name = str8_prefix(layer_folder, layer_slash_pos);\ -test_infos[test_infos_count].layer = layer_name;\ -test_infos[test_infos_count].label = str8_lit(#name);\ -test_infos[test_infos_count].decl_line = (line);\ -test_infos[test_infos_count].skip = (skip_);\ -test_infos[test_infos_count].test_fn = test__##name;\ -test_infos_count += 1;\ -} + +internal void base_register_test(char *func_name, TestFunctionType *fn, char *file_path, int line, int skip); + +#define AddTest(name, file_path, line, skip_, ...) \ + TEST_FUNCTION_DEF(name); \ + __VA_ARGS__ void add_test__##name(void) { base_register_test(Stringify(name), test__##name, file_path, line, skip_); } + # if COMPILER_MSVC # pragma section(".CRT$XCU", read) -# define DeclareTest(name, skip) \ -AddTest(name, __FILE__, __LINE__, (skip))\ -__declspec(allocate(".CRT$XCU")) void (*add_test_ptr__##name)(void) = add_test__##name;\ -__pragma(comment(linker, "/include:" Stringify(add_test_ptr__##name))) +# define DeclareTest(name, skip) \ + /* register test */ AddTest(name, __FILE__, __LINE__, (skip)) \ + /* alloc function pointer */ __declspec(allocate(".CRT$XCU")) void (*add_test_ptr__##name)(void) = add_test__##name; \ + /* do not GC test caller */ __pragma(comment(linker, "/include:" Stringify(add_test_ptr__##name))) # elif COMPILER_GCC || COMPILER_CLANG +// clang and gcc allocate memory for the function pointer automatically # define DeclareTest(name, skip) AddTest(name, __FILE__, __LINE__, (skip), __attribute__((constructor))) # else # error DeclareTest not defined for this compiler. @@ -82,23 +76,22 @@ __pragma(comment(linker, "/include:" Stringify(add_test_ptr__##name))) # define DeclareTest(name, skip) #endif -#define Test(name) DeclareTest(name, 0)\ -TEST_FUNCTION_DEF(name) -#define SkippedTest(name) DeclareTest(name, 1)\ -TEST_FUNCTION_DEF(name) +#define Test(name) DeclareTest(name, 0) TEST_FUNCTION_DEF(name) +#define SkippedTest(name) DeclareTest(name, 1) TEST_FUNCTION_DEF(name) -#define test_out(string) str8_list_push(arena, ctx->test_out, (string)) -#define test_outf(...) str8_list_pushf(arena, ctx->test_out, __VA_ARGS__) - -#define TestCheck(c) do { if (!(c)) {\ -ctx->result_out[0] = (TestResult){ .fail_file = __FILE__, .fail_line = __LINE__, .fail_cond = Stringify(c) };\ -if(debugger_is_attached()) { Trap(); }\ -return;\ +#define TestCheck(c) do { if (!(c)) { \ + /* record failed check */ ctx->result_out[0] = (TestResult){ .fail_file = __FILE__, .fail_line = __LINE__, .fail_cond = Stringify(c) }; \ + /* under debugger? -> trap */ if(debugger_is_attached()) { Trap(); } \ + /* exit test */ return; \ } } while(0) -#define TestSkip() do {\ -ctx->result_out[0] = (TestResult){ .status = TestStatus_Skip };\ -return;\ +#define TestSkip() do { \ + ctx->result_out[0] = (TestResult){ .status = TestStatus_Skip }; \ + return; \ } while(0) +// test log +#define test_out(string) str8_list_push(arena, ctx->test_out, (string)) +#define test_outf(...) str8_list_pushf(arena, ctx->test_out, __VA_ARGS__) + #endif // BASE_TEST_H diff --git a/src/torture/torture.c b/src/torture/torture.c index d50a5426..fa48d96d 100644 --- a/src/torture/torture.c +++ b/src/torture/torture.c @@ -40,7 +40,7 @@ t_name_from_test_info(Arena *arena, TestInfo *test_info) } internal String8List -t_test_group_from_name(Arena *arena, String8 pattern) +t_test_layer_from_name(Arena *arena, String8 pattern) { Temp scratch = scratch_begin(&arena, 1); String8List matches = {0}; @@ -120,6 +120,7 @@ t_delete_dir(String8 path) if (info.props.flags & FilePropertyFlag_IsFolder) { t_delete_dir(str8f(scratch.arena, "%S/%S", path, info.name)); + continue; } @@ -1030,7 +1031,7 @@ t_help(void) fprintf(stderr, " Usage: torture [Options] \n\n"); fprintf(stderr, " Options:\n"); fprintf(stderr, " -list Print available test targets\n"); - fprintf(stderr, " --gui Launch debugger with window\n"); + fprintf(stderr, " -gui Launch debugger with window\n"); fprintf(stderr, " -cl: Override default cl path\n"); fprintf(stderr, " -clang: Override default clang path\n"); fprintf(stderr, " -gcc: Override default gcc path\n"); @@ -1155,17 +1156,12 @@ t_entry_point(CmdLine *cmdline) String8 test_name = t_name_from_test_info(scratch.arena, test_info); if (str8_match_wildcard(test_name, t, StringMatchFlag_CaseInsensitive)) { - // TODO(rjf): do we really need to mutate the test infos here? this test won't - // even run, so I am not sure what setting the bit does - we already collect - // the tests that we *will* run -#if 0 // set skip flag switch (mode) { case Mode_Default: break; - case Mode_Skip: g_torture_tests[test_idx]->skip = 1; break; - case Mode_Force: g_torture_tests[test_idx]->skip = 0; break; + case Mode_Skip: test_info->skip = 1; break; + case Mode_Force: test_info->skip = 0; break; } -#endif // append test when not in skipping mode if ( ! hash_map_search_string_u64(&hm, test_name)) { @@ -1238,12 +1234,12 @@ t_entry_point(CmdLine *cmdline) U64Array target_indices = u64_array_from_list(scratch.arena, &targets); U64 max_label_size = 0; - U64 max_group_size = 0; + U64 max_layer_size = 0; for EachIndex(i, target_indices.count) { U64 test_idx = target_indices.v[i]; TestInfo *test_info = g_sorted_test_infos[test_idx]; max_label_size = Max(max_label_size, test_info->label.size); - max_group_size = Max(max_group_size, test_info->layer.size); + max_layer_size = Max(max_layer_size, test_info->layer.size); } U64 run_counters[TestStatus_COUNT] = {0}; @@ -1258,7 +1254,7 @@ t_entry_point(CmdLine *cmdline) for EachIndex(i, target_indices.count) { if (i == 0) { PrintHeader("Tests"); } - + U64 target_idx = target_indices.v[i]; TestInfo *test = g_sorted_test_infos[target_idx]; @@ -1268,7 +1264,7 @@ t_entry_point(CmdLine *cmdline) U64 curr_digit_count = count_digits_u64(i+1, 10); int idx_align_space_count = (int)(max_digit_count - curr_digit_count); fprintf(stdout, "[%.*s%llu/%llu] ", idx_align_space_count, spaces, (unsigned long long)i+1, (unsigned long long)target_indices.count); - fprintf(stdout, "%.*s %.*s/ %.*s", str8_varg(test->layer), (int)(max_group_size - test->layer.size), spaces, str8_varg(test->label)); + fprintf(stdout, "%.*s %.*s/ %.*s", str8_varg(test->layer), (int)(max_layer_size - test->layer.size), spaces, str8_varg(test->label)); fprintf(stdout, " %.*s ", (int)dots_count, dots); fflush(stdout); @@ -1375,6 +1371,7 @@ t_entry_point(CmdLine *cmdline) fprintf(stderr, " Skipped %llu\n", (unsigned long long)run_counters[TestStatus_Skip]); fprintf(stderr, " Time %.*s\n", str8_varg(total_time_str)); + // count slow tests U64 slow_count = 0; for EachElement(i, slowest) { Slowest s = slowest[i]; @@ -1382,26 +1379,28 @@ t_entry_point(CmdLine *cmdline) slow_count += 1; } - if (slow_count > 3) { + if (slow_count > 1) { U64 label_max = 0; - U64 group_max = 0; + U64 layer_max = 0; for EachElement(i, slowest) { Slowest s = slowest[i]; label_max = Max(g_sorted_test_infos[s.target_idx]->label.size, label_max); - group_max = Max(g_sorted_test_infos[s.target_idx]->layer.size, group_max); + layer_max = Max(g_sorted_test_infos[s.target_idx]->layer.size, layer_max); } fprintf(stderr, " \nSlow Tests\n"); for EachElement(i, slowest) { Slowest s = slowest[i]; if (s.target_idx >= test_infos_count) { break; } - TestInfo *test_info = g_sorted_test_infos[i]; - String8 elapsed_time = string_from_elapsed_time(scratch.arena, date_time_from_micro_seconds(s.d)); + + TestInfo *test_info = g_sorted_test_infos[s.target_idx]; + String8 elapsed_time = string_from_elapsed_time(scratch.arena, date_time_from_micro_seconds(s.d)); + fprintf(stderr, " %.*s %.*s/ %.*s %.*s %.*s\n", str8_varg(test_info->layer), - (int)(group_max - test_info->layer.size), spaces, + (int)(layer_max - test_info->layer.size), spaces, str8_varg(test_info->label), - (int)(label_max - test_info->layer.size) + 4, dots, + (int)(label_max - test_info->label.size) + 4, dots, str8_varg(elapsed_time)); } } diff --git a/src/torture/torture.h b/src/torture/torture.h index 78b8df15..56578809 100644 --- a/src/torture/torture.h +++ b/src/torture/torture.h @@ -5,12 +5,6 @@ //////////////////////////////// -#define T_RESET "\x1b[0m" -#define T_RED "\x1b[31m" -#define T_GREEN "\x1b[32m" -#define T_YELLOW "\x1b[33m" -#define T_BLUE "\x1b[34m" - typedef struct { TestInfo *test; @@ -19,14 +13,7 @@ typedef struct TestResult result; } T_RunCtx; -#define TEST(name) Test(name) -#define SKIP(name) SkippedTest(name) -#define T_Ok(c) TestCheck(c) - -#define T_MatchLinef(out, ...) T_Ok(t_match_linef(out, __VA_ARGS__)) -#define t_outf(...) str8_list_pushf(arena, ctx->test_out, ## __VA_ARGS__) - -//////////////////////////////////////////////////////////////// +//////////////////////////////// #define TIMEOUT_US(x) (x) #define TIMEOUT_MS(x) TIMEOUT_US((x)*1000ull) @@ -36,11 +23,19 @@ typedef struct #define ENDT_MS(x) TIMEOUT_US((x)*1000ull) #define ENDT_SEC(x) TIMEOUT_MS((x)*1000ull) -//////////////////////////////////////////////////////////////// +//////////////////////////////// -// output directory -internal B32 t_write_file_list(String8 name, String8List data); -internal B32 t_write_file(String8 name, String8 data); +#define T_RESET "\x1b[0m" +#define T_RED "\x1b[31m" +#define T_GREEN "\x1b[32m" +#define T_YELLOW "\x1b[33m" +#define T_BLUE "\x1b[34m" + +//////////////////////////////// + +// artifacts directory helpers +internal B32 t_write_file_list(String8 name, String8List data); +internal B32 t_write_file(String8 name, String8 data); internal String8 t_read_file(Arena *arena, String8 name); internal B32 t_delete_file(String8 name); internal String8 t_make_file_path(Arena *arena, String8 name); @@ -59,6 +54,7 @@ internal String8 t_radlink_path(void); internal String8 t_cwd_path(void); internal String8 t_src_path(void); +// process helpers internal B32 t_invoke(String8 exe, String8 cmdline, U64 timeout); internal B32 t_invoke_cl(char *fmt, ...); internal B32 t_invoke_linkerf(char *fmt, ...); @@ -68,16 +64,18 @@ internal void t_kill_all(String8 pattern); #define t_invoke_linker_timeoutf(t, f, ...) t_invoke_linker_timeout(push_str8f(arena, f, ##__VA_ARGS__), t) #define t_invoke_linker(c) t_invoke_linker_timeout(c, max_U64) -internal void t_kill_all(String8 pattern); - internal String8 t_chop_line(String8 *string); -internal B32 t_match_line(String8 *output, String8 expected_line); -internal B32 t_match_linef(String8 *output, char *fmt, ...); +internal B32 t_match_line(String8 *output, String8 expected_line); +internal B32 t_match_linef(String8 *output, char *fmt, ...); +#define T_MatchLinef(out, ...) T_Ok(t_match_linef(out, __VA_ARGS__)) // files helper internal String8List t_file_paths_from_dir(Arena *arena, String8 dir); -// printer +// TODO: deprecate internal void t_infof(char *fmt, ...); internal void t_errorf(char *fmt, ...); +#define TEST(name) Test(name) +#define SKIP(name) SkippedTest(name) +#define T_Ok(c) TestCheck(c)