move breakpad converter to async layer; eliminate task system

This commit is contained in:
Ryan Fleury
2024-11-01 15:18:40 -07:00
parent aded54c97f
commit 4164b30aac
9 changed files with 20 additions and 3263 deletions
+2 -2
View File
@@ -241,6 +241,8 @@ not depend on any other layers in the codebase. The folders which contain these
layers are prefixed with `lib_`, like `lib_rdi_format`.
A list of the layers in the codebase and their associated namespaces is below:
- `async` (`ASYNC_`): Implements a system for asynchronous work to be queued
and executed on a thread pool.
- `base` (no namespace): Universal, codebase-wide constructs. Strings, math,
memory allocators, helper macros, command-line parsing, and so on. Depends
on no other codebase layers.
@@ -375,8 +377,6 @@ A list of the layers in the codebase and their associated namespaces is below:
as-needed basis. Higher level drawing features are implemented in the `draw`
layer.
- `scratch` (no namespace): Scratch space for small and transient test programs.
- `task_system` (`TS_`): Implements a system for kicking off asynchronous tasks
and obtaining artifacts from those tasks once they've completed.
- `texture_cache` (`TEX_`): Implements an asynchronously-filled cache for GPU
texture data, filled by data sourced in the `hash_store` layer's cache. Used
for asynchronously preparing data for visualization.
-3
View File
@@ -33,9 +33,6 @@ main_thread_base_entry_point(void (*entry_point)(CmdLine *cmdline), char **argum
#if defined(RDI_FROM_PDB_H) && !defined(P2R_INIT_MANUAL)
p2r_init();
#endif
#if defined(TASK_SYSTEM_H) && !defined(TS_INIT_MANUAL)
ts_init();
#endif
#if defined(HASH_STORE_H) && !defined(HS_INIT_MANUAL)
hs_init();
#endif
-2
View File
@@ -574,7 +574,6 @@
#include "base/base_inc.h"
#include "os/os_inc.h"
#include "async/async.h"
#include "task_system/task_system.h"
#include "rdi_format/rdi_format_local.h"
#include "rdi_make/rdi_make_local.h"
#include "mdesk/mdesk.h"
@@ -617,7 +616,6 @@
#include "base/base_inc.c"
#include "os/os_inc.c"
#include "async/async.c"
#include "task_system/task_system.c"
#include "rdi_format/rdi_format_local.c"
#include "rdi_make/rdi_make_local.c"
#include "mdesk/mdesk.c"
@@ -25,7 +25,7 @@
//- rjf: [h]
#include "base/base_inc.h"
#include "os/os_inc.h"
#include "task_system/task_system.h"
#include "async/async.h"
#include "rdi_make/rdi_make_local.h"
#include "coff/coff.h"
#include "codeview/codeview.h"
@@ -40,7 +40,7 @@
//- rjf: [c]
#include "base/base_inc.c"
#include "os/os_inc.c"
#include "task_system/task_system.c"
#include "async/async.c"
#include "rdi_make/rdi_make_local.c"
#include "coff/coff.c"
#include "codeview/codeview.c"
@@ -63,9 +63,10 @@ struct P2B_BakeUnitVMapIn
RDIM_UnitChunkList *units;
};
internal TS_TASK_FUNCTION_DEF(p2b_bake_unit_vmap_task__entry_point)
ASYNC_WORK_DEF(p2b_bake_unit_vmap_work)
{
P2B_BakeUnitVMapIn *in = (P2B_BakeUnitVMapIn *)p;
Arena *arena = p2r_state->work_thread_arenas[thread_idx];
P2B_BakeUnitVMapIn *in = (P2B_BakeUnitVMapIn *)input;
RDIM_UnitVMapBakeResult *out = push_array(arena, RDIM_UnitVMapBakeResult, 1);
*out = rdim_bake_unit_vmap(arena, in->units);
return out;
@@ -79,9 +80,10 @@ struct P2B_BakeLineTablesIn
RDIM_LineTableChunkList *line_tables;
};
internal TS_TASK_FUNCTION_DEF(p2b_bake_line_table_task__entry_point)
ASYNC_WORK_DEF(p2b_bake_line_table_work)
{
P2B_BakeLineTablesIn *in = (P2B_BakeLineTablesIn *)p;
Arena *arena = p2r_state->work_thread_arenas[thread_idx];
P2B_BakeLineTablesIn *in = (P2B_BakeLineTablesIn *)input;
RDIM_LineTableBakeResult *out = push_array(arena, RDIM_LineTableBakeResult, 1);
*out = rdim_bake_line_tables(arena, in->line_tables);
return out;
@@ -100,9 +102,10 @@ struct P2B_DumpProcChunkIn
RDIM_SymbolChunkNode *chunk;
};
internal TS_TASK_FUNCTION_DEF(p2b_dump_proc_chunk_task__entry_point)
ASYNC_WORK_DEF(p2b_dump_proc_chunk_work)
{
P2B_DumpProcChunkIn *in = (P2B_DumpProcChunkIn *)p;
Arena *arena = p2r_state->work_thread_arenas[thread_idx];
P2B_DumpProcChunkIn *in = (P2B_DumpProcChunkIn *)input;
String8List *out = push_array(arena, String8List, 1);
RDI_LineTable *line_tables = in->line_tables_bake->line_tables;
RDI_U64 line_tables_count = in->line_tables_bake->line_tables_count;
@@ -225,11 +228,11 @@ entry_point(CmdLine *cmdline)
//- rjf: kick off unit vmap baking
P2B_BakeUnitVMapIn bake_unit_vmap_in = {&params->units};
TS_Ticket bake_unit_vmap_ticket = ts_kickoff(p2b_bake_unit_vmap_task__entry_point, &bake_unit_vmap_in);
ASYNC_Task *bake_unit_vmap_task = async_task_launch(arena, p2b_bake_unit_vmap_work, .input = &bake_unit_vmap_in);
//- rjf: kick off line-table baking
P2B_BakeLineTablesIn bake_line_tables_in = {&params->line_tables};
TS_Ticket bake_line_tables_ticket = ts_kickoff(p2b_bake_line_table_task__entry_point, &bake_line_tables_in);
ASYNC_Task *bake_line_tables_task = async_task_launch(arena, p2b_bake_line_table_work, .input = &bake_line_tables_in);
//- rjf: build unit -> line table idx array
U64 unit_count = params->units.total_count;
@@ -264,19 +267,19 @@ entry_point(CmdLine *cmdline)
//- rjf: join unit vmap
ProfBegin("join unit vmap");
RDIM_UnitVMapBakeResult *bake_unit_vmap_out = ts_join_struct(bake_unit_vmap_ticket, max_U64, RDIM_UnitVMapBakeResult);
RDIM_UnitVMapBakeResult *bake_unit_vmap_out = async_task_join_struct(bake_unit_vmap_task, RDIM_UnitVMapBakeResult);
RDI_VMapEntry *unit_vmap = bake_unit_vmap_out->vmap.vmap;
U32 unit_vmap_count = bake_unit_vmap_out->vmap.count;
ProfEnd();
//- rjf: join line tables
ProfBegin("join line table");
RDIM_LineTableBakeResult *bake_line_tables_out = ts_join_struct(bake_line_tables_ticket, max_U64, RDIM_LineTableBakeResult);
RDIM_LineTableBakeResult *bake_line_tables_out = async_task_join_struct(bake_line_tables_task, RDIM_LineTableBakeResult);
ProfEnd();
//- rjf: kick off FUNC & line record dump tasks
P2B_DumpProcChunkIn *dump_proc_chunk_in = push_array(arena, P2B_DumpProcChunkIn, params->procedures.chunk_count);
TS_Ticket *dump_proc_chunk_tickets = push_array(arena, TS_Ticket, params->procedures.chunk_count);
ASYNC_Task **dump_proc_chunk_tasks = push_array(arena, ASYNC_Task *, params->procedures.chunk_count);
ProfScope("kick off FUNC & line record dump tasks")
{
U64 task_idx = 0;
@@ -288,7 +291,7 @@ entry_point(CmdLine *cmdline)
dump_proc_chunk_in[task_idx].unit_count = unit_count;
dump_proc_chunk_in[task_idx].line_tables_bake = bake_line_tables_out;
dump_proc_chunk_in[task_idx].chunk = n;
dump_proc_chunk_tickets[task_idx] = ts_kickoff(p2b_dump_proc_chunk_task__entry_point, &dump_proc_chunk_in[task_idx]);
dump_proc_chunk_tasks[task_idx] = async_task_launch(arena, p2b_dump_proc_chunk_work, .input = &dump_proc_chunk_in[task_idx]);
}
}
@@ -297,7 +300,7 @@ entry_point(CmdLine *cmdline)
{
for(U64 idx = 0; idx < params->procedures.chunk_count; idx += 1)
{
String8List *out = ts_join_struct(dump_proc_chunk_tickets[idx], max_U64, String8List);
String8List *out = async_task_join_struct(dump_proc_chunk_tasks[idx], String8List);
str8_list_concat_in_place(&dump, out);
}
}
File diff suppressed because it is too large Load Diff
-2
View File
@@ -24,7 +24,6 @@
#include "base/base_inc.h"
#include "os/os_inc.h"
#include "async/async.h"
// #include "task_system/task_system.h"
#include "rdi_make/rdi_make_local.h"
#include "coff/coff.h"
#include "codeview/codeview.h"
@@ -40,7 +39,6 @@
#include "base/base_inc.c"
#include "os/os_inc.c"
#include "async/async.c"
//#include "task_system/task_system.c"
#include "rdi_make/rdi_make_local.c"
#include "coff/coff.c"
#include "codeview/codeview.c"
-2
View File
@@ -19,7 +19,6 @@
//- rjf: [h]
#include "base/base_inc.h"
#include "os/os_inc.h"
#include "task_system/task_system.h"
#include "rdi_make/rdi_make_local.h"
#include "coff/coff.h"
#include "codeview/codeview.h"
@@ -33,7 +32,6 @@
//- rjf: [c]
#include "base/base_inc.c"
#include "os/os_inc.c"
#include "task_system/task_system.c"
#include "rdi_make/rdi_make_local.c"
#include "coff/coff.c"
#include "codeview/codeview.c"
-238
View File
@@ -1,238 +0,0 @@
// Copyright (c) 2024 Epic Games Tools
// Licensed under the MIT license (https://opensource.org/license/mit/)
////////////////////////////////
//~ rjf: Basic Type Functions
internal TS_Ticket
ts_ticket_zero(void)
{
TS_Ticket ticket = {0};
return ticket;
}
internal B32
ts_ticket_match(TS_Ticket a, TS_Ticket b)
{
B32 result = MemoryMatchStruct(&a, &b);
return result;
}
internal void
ts_ticket_list_push(Arena *arena, TS_TicketList *list, TS_Ticket ticket)
{
TS_TicketNode *n = push_array(arena, TS_TicketNode, 1);
n->v = ticket;
SLLQueuePush(list->first, list->last, n);
list->count += 1;
}
////////////////////////////////
//~ rjf: Top-Level Layer Initialization
internal void
ts_init(void)
{
Arena *arena = arena_alloc();
ts_shared = push_array(arena, TS_Shared, 1);
ts_shared->arena = arena;
ts_shared->artifact_slots_count = 1024;
ts_shared->artifact_stripes_count = Min(ts_shared->artifact_slots_count, os_get_system_info()->logical_processor_count);
ts_shared->artifact_slots = push_array(arena, TS_TaskArtifactSlot, ts_shared->artifact_slots_count);
ts_shared->artifact_stripes = push_array(arena, TS_TaskArtifactStripe, ts_shared->artifact_stripes_count);
for(U64 idx = 0; idx < ts_shared->artifact_stripes_count; idx += 1)
{
ts_shared->artifact_stripes[idx].arena = arena_alloc();
ts_shared->artifact_stripes[idx].cv = os_condition_variable_alloc();
ts_shared->artifact_stripes[idx].rw_mutex = os_rw_mutex_alloc();
}
ts_shared->u2t_ring_size = MB(1);
ts_shared->u2t_ring_base = push_array_no_zero(arena, U8, ts_shared->u2t_ring_size);
ts_shared->u2t_ring_mutex = os_mutex_alloc();
ts_shared->u2t_ring_cv = os_condition_variable_alloc();
ts_shared->task_threads_count = Max(1, os_get_system_info()->logical_processor_count-1);
ts_shared->task_threads = push_array(arena, TS_TaskThread, ts_shared->task_threads_count);
for(U64 idx = 0; idx < ts_shared->task_threads_count; idx += 1)
{
ts_shared->task_threads[idx].arena = arena_alloc();
ts_shared->task_threads[idx].thread = os_thread_launch(ts_task_thread__entry_point, (void *)idx, 0);
}
}
////////////////////////////////
//~ rjf: Top-Level Accessors
internal U64
ts_thread_count(void)
{
return ts_shared->task_threads_count;
}
////////////////////////////////
//~ rjf: High-Level Task Kickoff / Joining
internal TS_Ticket
ts_kickoff_(TS_TaskFunctionType *entry_point, TS_KickoffParams *params)
{
ProfBeginFunction();
// rjf: obtain number & slot/stripe for next artifact
U64 artifact_num = ins_atomic_u64_inc_eval(&ts_shared->artifact_num_gen);
U64 slot_idx = artifact_num%ts_shared->artifact_slots_count;
U64 stripe_idx = slot_idx%ts_shared->artifact_stripes_count;
TS_TaskArtifactSlot *slot = &ts_shared->artifact_slots[slot_idx];
TS_TaskArtifactStripe *stripe = &ts_shared->artifact_stripes[stripe_idx];
// rjf: allocate artifact
TS_TaskArtifact *artifact = 0;
ProfScope("allocate artifact")
{
OS_MutexScopeW(stripe->rw_mutex)
{
artifact = stripe->free_artifact;
if(artifact != 0)
{
SLLStackPop(stripe->free_artifact);
}
else
{
artifact = push_array_no_zero(stripe->arena, TS_TaskArtifact, 1);
}
artifact->num = artifact_num;
artifact->task_is_done = 0;
artifact->out = 0;
}
}
// rjf: form ticket out of artifact info
TS_Ticket ticket = {artifact_num, (U64)artifact};
// rjf: push task info to task ring buffer
ProfScope("push task info to task ring buffer")
{
OS_MutexScope(ts_shared->u2t_ring_mutex) for(;;)
{
U64 unconsumed_size = ts_shared->u2t_ring_write_pos - ts_shared->u2t_ring_read_pos;
U64 available_size = ts_shared->u2t_ring_size-unconsumed_size;
if(available_size >= sizeof(entry_point) + sizeof(Arena *) + sizeof(params->in) + sizeof(ticket))
{
Arena *task_arena = 0;
if(params->optional_arena_ptr != 0)
{
task_arena = *params->optional_arena_ptr;
}
ts_shared->u2t_ring_write_pos += ring_write_struct(ts_shared->u2t_ring_base, ts_shared->u2t_ring_size, ts_shared->u2t_ring_write_pos, &entry_point);
ts_shared->u2t_ring_write_pos += ring_write_struct(ts_shared->u2t_ring_base, ts_shared->u2t_ring_size, ts_shared->u2t_ring_write_pos, &task_arena);
ts_shared->u2t_ring_write_pos += ring_write_struct(ts_shared->u2t_ring_base, ts_shared->u2t_ring_size, ts_shared->u2t_ring_write_pos, &params->in);
ts_shared->u2t_ring_write_pos += ring_write_struct(ts_shared->u2t_ring_base, ts_shared->u2t_ring_size, ts_shared->u2t_ring_write_pos, &ticket);
if(params->optional_arena_ptr != 0)
{
*params->optional_arena_ptr = 0;
}
break;
}
os_condition_variable_wait(ts_shared->u2t_ring_cv, ts_shared->u2t_ring_mutex, max_U64);
}
os_condition_variable_signal(ts_shared->u2t_ring_cv);
}
ProfEnd();
return ticket;
}
internal TS_JoinResult
ts_join(TS_Ticket ticket, U64 endt_us)
{
TS_JoinResult result = {0};
U64 artifact_num = ticket.u64[0];
U64 slot_idx = artifact_num%ts_shared->artifact_slots_count;
U64 stripe_idx = slot_idx%ts_shared->artifact_stripes_count;
TS_TaskArtifactSlot *slot = &ts_shared->artifact_slots[slot_idx];
TS_TaskArtifactStripe *stripe = &ts_shared->artifact_stripes[stripe_idx];
TS_TaskArtifact *artifact = (TS_TaskArtifact *)ticket.u64[1];
if(artifact != 0)
{
OS_MutexScopeR(stripe->rw_mutex) for(;;)
{
B64 task_is_done = artifact->task_is_done;
if(task_is_done)
{
OS_MutexScopeRWPromote(stripe->rw_mutex)
{
result.good = 1;
result.out = artifact->out;
SLLStackPush(stripe->free_artifact, artifact);
}
break;
}
if(os_now_microseconds() >= endt_us)
{
break;
}
os_condition_variable_wait_rw_r(stripe->cv, stripe->rw_mutex, endt_us);
}
}
return result;
}
////////////////////////////////
//~ rjf: Task Threads
internal void
ts_u2t_dequeue_task(TS_TaskFunctionType **entry_point_out, Arena **arena_out, void **p_out, TS_Ticket *ticket_out)
{
OS_MutexScope(ts_shared->u2t_ring_mutex) for(;;)
{
U64 unconsumed_size = ts_shared->u2t_ring_write_pos - ts_shared->u2t_ring_read_pos;
if(unconsumed_size >= sizeof(*entry_point_out) + sizeof(*p_out) + sizeof(*ticket_out))
{
ts_shared->u2t_ring_read_pos += ring_read_struct(ts_shared->u2t_ring_base, ts_shared->u2t_ring_size, ts_shared->u2t_ring_read_pos, entry_point_out);
ts_shared->u2t_ring_read_pos += ring_read_struct(ts_shared->u2t_ring_base, ts_shared->u2t_ring_size, ts_shared->u2t_ring_read_pos, arena_out);
ts_shared->u2t_ring_read_pos += ring_read_struct(ts_shared->u2t_ring_base, ts_shared->u2t_ring_size, ts_shared->u2t_ring_read_pos, p_out);
ts_shared->u2t_ring_read_pos += ring_read_struct(ts_shared->u2t_ring_base, ts_shared->u2t_ring_size, ts_shared->u2t_ring_read_pos, ticket_out);
break;
}
os_condition_variable_wait(ts_shared->u2t_ring_cv, ts_shared->u2t_ring_mutex, max_U64);
}
os_condition_variable_broadcast(ts_shared->u2t_ring_cv);
}
internal void
ts_task_thread__entry_point(void *p)
{
U64 thread_idx = (U64)p;
ThreadNameF("[ts] task thread #%I64u", thread_idx);
TS_TaskThread *thread = &ts_shared->task_threads[thread_idx];
for(;;)
{
//- rjf: grab next task
TS_TaskFunctionType *task_function = 0;
Arena *task_arena = 0;
void *task_params = 0;
TS_Ticket task_ticket = {0};
ts_u2t_dequeue_task(&task_function, &task_arena, &task_params, &task_ticket);
//- rjf: use task thread's arena if none specified
if(task_arena == 0)
{
task_arena = thread->arena;
}
//- rjf: run task
void *task_out = task_function(task_arena, thread_idx, task_params);
//- rjf: store into artifact
U64 artifact_num = task_ticket.u64[0];
U64 slot_idx = artifact_num%ts_shared->artifact_slots_count;
U64 stripe_idx = slot_idx%ts_shared->artifact_stripes_count;
TS_TaskArtifactSlot *slot = &ts_shared->artifact_slots[slot_idx];
TS_TaskArtifactStripe *stripe = &ts_shared->artifact_stripes[stripe_idx];
TS_TaskArtifact *artifact = (TS_TaskArtifact *)task_ticket.u64[1];
OS_MutexScopeW(stripe->rw_mutex)
{
artifact->task_is_done = 1;
artifact->out = task_out;
}
os_condition_variable_broadcast(stripe->cv);
}
}
-162
View File
@@ -1,162 +0,0 @@
// Copyright (c) 2024 Epic Games Tools
// Licensed under the MIT license (https://opensource.org/license/mit/)
#ifndef TASK_SYSTEM_H
#define TASK_SYSTEM_H
////////////////////////////////
//~ rjf: Task "Ticket" Type
//
// "Tickets" are opaque handles, used to refer to submitted tasks.
//
typedef struct TS_Ticket TS_Ticket;
struct TS_Ticket
{
U64 u64[2];
};
typedef struct TS_TicketNode TS_TicketNode;
struct TS_TicketNode
{
TS_TicketNode *next;
TS_Ticket v;
};
typedef struct TS_TicketList TS_TicketList;
struct TS_TicketList
{
TS_TicketNode *first;
TS_TicketNode *last;
U64 count;
};
////////////////////////////////
//~ rjf: Task Request Types
#define TS_TASK_FUNCTION_DEF(name) void *name(Arena *arena, U64 thread_idx, void *p)
typedef TS_TASK_FUNCTION_DEF(TS_TaskFunctionType);
////////////////////////////////
//~ rjf: Task Kickoff Parameters
typedef struct TS_KickoffParams TS_KickoffParams;
struct TS_KickoffParams
{
Arena **optional_arena_ptr;
void *in;
};
////////////////////////////////
//~ rjf: Task Join Result
typedef struct TS_JoinResult TS_JoinResult;
struct TS_JoinResult
{
B32 good;
void *out;
};
////////////////////////////////
//~ rjf: Task Artifact Cache Types
typedef struct TS_TaskArtifact TS_TaskArtifact;
struct TS_TaskArtifact
{
TS_TaskArtifact *next;
U64 num;
B64 task_is_done;
void *out;
};
typedef struct TS_TaskArtifactSlot TS_TaskArtifactSlot;
struct TS_TaskArtifactSlot
{
TS_TaskArtifact *first;
TS_TaskArtifact *last;
};
typedef struct TS_TaskArtifactStripe TS_TaskArtifactStripe;
struct TS_TaskArtifactStripe
{
Arena *arena;
OS_Handle cv;
OS_Handle rw_mutex;
TS_TaskArtifact *free_artifact;
};
////////////////////////////////
//~ rjf: Per-Thread State
typedef struct TS_TaskThread TS_TaskThread;
struct TS_TaskThread
{
Arena *arena;
OS_Handle thread;
};
////////////////////////////////
//~ rjf: Main Shared State
typedef struct TS_Shared TS_Shared;
struct TS_Shared
{
Arena *arena;
// rjf: task artifact cache
U64 artifact_num_gen;
U64 artifact_slots_count;
U64 artifact_stripes_count;
TS_TaskArtifactSlot *artifact_slots;
TS_TaskArtifactStripe *artifact_stripes;
// rjf: task ring buffer
U64 u2t_ring_size;
U8 *u2t_ring_base;
U64 u2t_ring_write_pos;
U64 u2t_ring_read_pos;
OS_Handle u2t_ring_mutex;
OS_Handle u2t_ring_cv;
// rjf: task threads
TS_TaskThread *task_threads;
U64 task_threads_count;
};
////////////////////////////////
//~ rjf: Globals
global TS_Shared *ts_shared = 0;
////////////////////////////////
//~ rjf: Basic Type Functions
internal TS_Ticket ts_ticket_zero(void);
internal B32 ts_ticket_match(TS_Ticket a, TS_Ticket b);
internal void ts_ticket_list_push(Arena *arena, TS_TicketList *list, TS_Ticket ticket);
////////////////////////////////
//~ rjf: Top-Level Layer Initialization
internal void ts_init(void);
////////////////////////////////
//~ rjf: Top-Level Accessors
internal U64 ts_thread_count(void);
////////////////////////////////
//~ rjf: High-Level Task Kickoff / Joining
internal TS_Ticket ts_kickoff_(TS_TaskFunctionType *entry_point, TS_KickoffParams *params);
#define ts_kickoff(entry_point, input_ptr, ...) ts_kickoff_((entry_point), &(TS_KickoffParams){.in = (input_ptr), __VA_ARGS__})
internal TS_JoinResult ts_join(TS_Ticket ticket, U64 endt_us);
#define ts_join_struct(ticket, endt_us, type) (type *)ts_join((ticket), (endt_us)).out
////////////////////////////////
//~ rjf: Task Threads
internal void ts_u2t_dequeue_task(TS_TaskFunctionType **entry_point_out, Arena **arena_out, void **p_out, TS_Ticket *ticket_out);
internal void ts_task_thread__entry_point(void *p);
#endif // TASK_SYSTEM_H