store partially-qualified names in generated RDIs; adjust debugger to do fully-qualified lookups when appropriate

This commit is contained in:
Ryan Fleury
2026-04-21 19:36:38 -07:00
parent 8cf14b5acb
commit 6f2b8540eb
6 changed files with 498 additions and 240 deletions
+301 -180
View File
@@ -314,7 +314,7 @@ di_open(DI_Key key)
internal void
di_close(DI_Key key, B32 force_closed)
{
{
//- rjf: unpack key
U64 hash = u64_hash_from_str8(str8_struct(&key));
U64 slot_idx = hash%di_shared->slots_count;
@@ -340,15 +340,15 @@ di_close(DI_Key key, B32 force_closed)
}
}
if(node)
{
if(force_closed)
{
node->refcount = 0;
}
else
{
node->refcount -= 1;
}
{
if(force_closed)
{
node->refcount = 0;
}
else
{
node->refcount -= 1;
}
if(node->refcount == 0)
{
for(;;)
@@ -374,9 +374,9 @@ di_close(DI_Key key, B32 force_closed)
//- rjf: release node's resources if needed
if(node_released)
{
ins_atomic_u64_dec_eval(&di_shared->load_count);
ins_atomic_u64_inc_eval(&di_shared->load_gen);
{
ins_atomic_u64_dec_eval(&di_shared->load_count);
ins_atomic_u64_inc_eval(&di_shared->load_gen);
os_file_map_view_close(file_map, file_base, r1u64(0, file_props.size));
os_file_map_close(file_map);
os_file_close(file);
@@ -384,7 +384,7 @@ di_close(DI_Key key, B32 force_closed)
{
arena_release(arena);
}
}
}
}
////////////////////////////////
@@ -396,13 +396,13 @@ di_load_gen(void)
U64 result = ins_atomic_u64_eval(&di_shared->load_gen);
return result;
}
internal U64
di_load_count(void)
{
internal U64
di_load_count(void)
{
U64 result = ins_atomic_u64_eval(&di_shared->load_count);
return result;
}
return result;
}
internal DI_KeyArray
di_push_all_loaded_keys(Arena *arena)
@@ -598,14 +598,14 @@ di_async_tick(void)
////////////////////////////
//- rjf: generate load tasks for all unique requests
//
for EachElement(priority_idx, first_req)
{
//
for EachElement(priority_idx, first_req)
{
for EachNode(n, DI_RequestNode, first_req[priority_idx])
{
// rjf: unpack request
DI_Key key = n->v.key;
// rjf: determine if this request is a duplicate
B32 request_is_duplicate = 1;
{
@@ -625,7 +625,7 @@ di_async_tick(void)
}
}
}
// rjf: if not a duplicate, create new task
if(!request_is_duplicate)
{
@@ -642,25 +642,25 @@ di_async_tick(void)
DLLPushBack(di_shared->first_load_task[priority_idx], di_shared->last_load_task[priority_idx], t);
t->key = key;
}
}
}
}
}
////////////////////////////
//- rjf: update tasks: configure, launch if we can, & retire if we can
//
for EachElement(priority_idx, di_shared->first_load_task)
{
//
for EachElement(priority_idx, di_shared->first_load_task)
{
for(DI_LoadTask *t = di_shared->first_load_task[priority_idx], *next = 0; t != 0; t = next)
{
next = t->next;
//- rjf: unpack key
DI_Key key = t->key;
U64 key_hash = u64_hash_from_str8(str8_struct(&key));
U64 key_slot_idx = key_hash%di_shared->key2path_slots_count;
DI_KeySlot *key_slot = &di_shared->key2path_slots[key_slot_idx];
Stripe *key_stripe = stripe_from_slot_idx(&di_shared->key2path_stripes, key_slot_idx);
//- rjf: get key's O.G. path
String8 og_path = {0};
U64 og_min_timestamp = 0;
@@ -676,13 +676,13 @@ di_async_tick(void)
}
}
}
//- rjf: analyze O.G. debug info
//- rjf: analyze O.G. debug info
if(!t->og_analyzed)
{
t->og_analyzed = 1;
OS_Handle file = os_file_open(OS_AccessFlag_ShareRead|OS_AccessFlag_Read, og_path);
FileProperties props = os_properties_from_file(file);
FileProperties props = os_properties_from_file(file);
t->og_size = props.size;
U64 rdi_magic_maybe = 0;
if(os_file_read_struct(file, 0, &rdi_magic_maybe) == 8 &&
@@ -692,12 +692,12 @@ di_async_tick(void)
}
os_file_close(file);
}
U64 og_size = t->og_size;
U64 og_size = t->og_size;
B32 og_is_rdi = t->og_is_rdi;
B32 og_is_good = (og_size > 0);
B32 og_is_good = (og_size > 0);
//- rjf: compute key's RDI path
String8 rdi_path = {0};
String8 rdi_path = {0};
{
if(og_is_rdi)
{
@@ -708,7 +708,7 @@ di_async_tick(void)
rdi_path = str8f(scratch.arena, "%S.rdi", str8_chop_last_dot(og_path));
}
}
//- rjf: determine if RDI is stale
if(!t->rdi_analyzed)
{
@@ -731,16 +731,16 @@ di_async_tick(void)
os_file_close(file);
}
B32 rdi_is_stale = t->rdi_is_stale;
//- rjf: calculate thread counts for conversion processes
if(!og_is_rdi && rdi_is_stale && t->thread_count == 0)
{
U64 thread_count = 1;
U64 max_thread_count = os_get_system_info()->logical_processor_count/2;
if(priority_idx > 0)
{
max_thread_count = Max(1, max_thread_count/2);
}
U64 max_thread_count = os_get_system_info()->logical_processor_count/2;
if(priority_idx > 0)
{
max_thread_count = Max(1, max_thread_count/2);
}
{
if(0){}
else if(og_size <= MB(4)) {thread_count = 1;}
@@ -752,7 +752,7 @@ di_async_tick(void)
thread_count = Max(1, thread_count);
t->thread_count = thread_count;
}
//- rjf: determine if there are threads available
B32 threads_available = 0;
{
@@ -761,30 +761,30 @@ di_async_tick(void)
U64 needed_threads = (current_threads + t->thread_count);
threads_available = (max_threads >= needed_threads);
}
//- rjf: if this conversion will overwrite an RDI we already have in cache,
// then we need to evict the old one from the cache.
B32 ready_to_launch_conversion = (threads_available && !og_is_rdi && rdi_is_stale && t->thread_count != 0 && t->status != DI_LoadTaskStatus_Active);
if(ready_to_launch_conversion)
{
//- rjf: if this conversion will overwrite an RDI we already have in cache,
// then we need to evict the old one from the cache.
B32 ready_to_launch_conversion = (threads_available && !og_is_rdi && rdi_is_stale && t->thread_count != 0 && t->status != DI_LoadTaskStatus_Active);
if(ready_to_launch_conversion)
{
U64 path2key_hash = u64_hash_from_str8(og_path);
U64 path2key_slot_idx = path2key_hash%di_shared->path2key_slots_count;
DI_KeySlot *path2key_slot = &di_shared->path2key_slots[path2key_slot_idx];
Stripe *path2key_stripe = stripe_from_slot_idx(&di_shared->path2key_stripes, path2key_slot_idx);
RWMutexScope(path2key_stripe->rw_mutex, 0)
{
// NOTE(rjf): we need to iterate from last -> first, since we want to evict the
// most recent key.
for(DI_KeyPathNode *n = path2key_slot->last; n != 0; n = n->prev)
{
if(str8_match(n->path, og_path, 0) && !di_key_match(key, n->key))
{
di_close(n->key, 1);
}
}
}
}
Stripe *path2key_stripe = stripe_from_slot_idx(&di_shared->path2key_stripes, path2key_slot_idx);
RWMutexScope(path2key_stripe->rw_mutex, 0)
{
// NOTE(rjf): we need to iterate from last -> first, since we want to evict the
// most recent key.
for(DI_KeyPathNode *n = path2key_slot->last; n != 0; n = n->prev)
{
if(str8_match(n->path, og_path, 0) && !di_key_match(key, n->key))
{
di_close(n->key, 1);
}
}
}
}
//- rjf: launch conversion processes
if(og_is_good && ready_to_launch_conversion)
{
@@ -812,7 +812,7 @@ di_async_tick(void)
t->status = DI_LoadTaskStatus_Active;
di_shared->conversion_process_count += 1;
di_shared->conversion_thread_count += t->thread_count;
// rjf: send event
MutexScope(di_shared->event_mutex)
{
@@ -823,7 +823,7 @@ di_async_tick(void)
n->v.string = str8_copy(di_shared->event_arena, rdi_path);
}
}
//- rjf: if active & process has completed, mark as done
{
U64 exit_code = 0;
@@ -849,21 +849,21 @@ di_async_tick(void)
di_shared->conversion_thread_count -= t->thread_count;
}
}
}
//- rjf: ready to launch, but bad O.G. file -> just immediately mark as done
if(!og_is_good && ready_to_launch_conversion)
{
t->status = DI_LoadTaskStatus_Done;
}
}
//- rjf: ready to launch, but bad O.G. file -> just immediately mark as done
if(!og_is_good && ready_to_launch_conversion)
{
t->status = DI_LoadTaskStatus_Done;
}
//- rjf: if the RDI for this task is not stale, then we're already done - mark this
// task as done & prepped for storing into the cache
if(!rdi_is_stale)
{
t->status = DI_LoadTaskStatus_Done;
}
//- rjf: if the RDI for this task *is* stale, but the O.G. path is actually RDI,
// then we can't actually re-convert to produce a non-stale RDI. in this case, just
// mark as done.
@@ -871,7 +871,7 @@ di_async_tick(void)
{
t->status = DI_LoadTaskStatus_Done;
}
//- rjf: if task is done, retire & recycle task; gather path to load
if(t->status == DI_LoadTaskStatus_Done)
{
@@ -891,8 +891,8 @@ di_async_tick(void)
SLLQueuePush(first_parse_task, last_parse_task, n);
parse_tasks_count += 1;
}
}
}
}
}
////////////////////////////
//- rjf: join all parse tasks
@@ -997,12 +997,12 @@ di_async_tick(void)
node->arena = rdi_parsed_arena;
MemoryCopyStruct(&node->rdi, &rdi_parsed);
node->completion_count += 1;
node->working_count -= 1;
if(node->rdi.raw_data_size != 0)
{
ins_atomic_u64_inc_eval(&di_shared->load_gen);
}
ins_atomic_u64_inc_eval(&di_shared->load_count);
node->working_count -= 1;
if(node->rdi.raw_data_size != 0)
{
ins_atomic_u64_inc_eval(&di_shared->load_gen);
}
ins_atomic_u64_inc_eval(&di_shared->load_count);
}
else
{
@@ -1122,16 +1122,16 @@ di_search_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out, U64 *
}
lane_sync();
//- rjf: do wide search on all lanes
Arena *arena = arena_alloc();
Arena **arenas = 0;
U64 arenas_count = lane_count();
if(lane_idx() == 0)
{
arenas = push_array(arena, Arena *, arenas_count);
}
lane_sync_u64(&arenas, 0);
arenas[lane_idx()] = arena;
//- rjf: do wide search on all lanes
Arena *arena = arena_alloc();
Arena **arenas = 0;
U64 arenas_count = lane_count();
if(lane_idx() == 0)
{
arenas = push_array(arena, Arena *, arenas_count);
}
lane_sync_u64(&arenas, 0);
arenas[lane_idx()] = arena;
DI_SearchItemChunkList *lanes_items = 0;
ProfScope("do wide search on all lanes")
{
@@ -1176,10 +1176,10 @@ di_search_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out, U64 *
Rng1U64 range = lane_range(element_count);
for EachInRange(idx, range)
{
//- rjf: every so often, check if we need to cancel, and cancel
if(idx%10000 == 0 && !!ins_atomic_u32_eval(cancel_signal))
//- rjf: every so often, check if we need to cancel, and cancel
if(idx%10000 == 0 && !!ins_atomic_u32_eval(cancel_signal))
{
break;
break;
}
//- rjf: get element, map to string; if empty, continue to next element
@@ -1279,15 +1279,15 @@ di_search_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out, U64 *
}
}
lane_sync();
//- rjf: decide if we cancelled
B32 cancelled = 0;
if(lane_idx() == 0 && !!ins_atomic_u32_eval(cancel_signal))
{
cancelled = 1;
}
lane_sync_u64(&cancelled, 0);
//- rjf: decide if we cancelled
B32 cancelled = 0;
if(lane_idx() == 0 && !!ins_atomic_u32_eval(cancel_signal))
{
cancelled = 1;
}
lane_sync_u64(&cancelled, 0);
//- rjf: produce sort records
typedef struct SortRecord SortRecord;
struct SortRecord
@@ -1444,20 +1444,20 @@ di_search_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out, U64 *
}
lane_sync();
//- rjf: bundle as artifact
if(!cancelled)
{
artifact.u64[0] = (U64)arenas;
artifact.u64[1] = arenas_count;
//- rjf: bundle as artifact
if(!cancelled)
{
artifact.u64[0] = (U64)arenas;
artifact.u64[1] = arenas_count;
artifact.u64[2] = (U64)items.v;
artifact.u64[3] = items.count;
}
//- rjf: release results on cancel
else
{
arena_release(arena);
}
artifact.u64[3] = items.count;
}
//- rjf: release results on cancel
else
{
arena_release(arena);
}
}
scratch_end(scratch);
access_close(access);
@@ -1467,20 +1467,20 @@ di_search_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out, U64 *
internal void
di_search_artifact_destroy(AC_Artifact artifact)
{
Temp scratch = scratch_begin(0, 0);
Arena **arenas = (Arena **)artifact.u64[0];
U64 arenas_count = artifact.u64[1];
Arena **arenas_copy = push_array(scratch.arena, Arena *, arenas_count);
MemoryCopy(arenas_copy, arenas, sizeof(Arena *) * arenas_count);
for EachIndex(idx, arenas_count)
{
if(arenas_copy[idx])
{
arena_release(arenas_copy[idx]);
}
}
scratch_end(scratch);
{
Temp scratch = scratch_begin(0, 0);
Arena **arenas = (Arena **)artifact.u64[0];
U64 arenas_count = artifact.u64[1];
Arena **arenas_copy = push_array(scratch.arena, Arena *, arenas_count);
MemoryCopy(arenas_copy, arenas, sizeof(Arena *) * arenas_count);
for EachIndex(idx, arenas_count)
{
if(arenas_copy[idx])
{
arena_release(arenas_copy[idx]);
}
}
scratch_end(scratch);
}
internal DI_SearchItemArray
@@ -1520,32 +1520,103 @@ di_match_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out, U64 *g
//- rjf: unpack key
U64 index = 0;
String8 name = {0};
DI_Key preferred_key = {0};
String8 name = {0};
DI_Key preferred_key = {0};
{
U64 key_read_off = 0;
key_read_off += str8_deserial_read_struct(key, key_read_off, &index);
key_read_off += str8_deserial_read_struct(key, key_read_off, &preferred_key);
key_read_off += str8_deserial_read_struct(key, key_read_off, &name.size);
key_read_off += str8_deserial_read_struct(key, key_read_off, &index);
key_read_off += str8_deserial_read_struct(key, key_read_off, &preferred_key);
key_read_off += str8_deserial_read_struct(key, key_read_off, &name.size);
name.str = push_array_no_zero(scratch.arena, U8, name.size);
key_read_off += str8_deserial_read(key, key_read_off, name.str, name.size, 1);
}
//- rjf: find split point between container part of name & "leaf"
U64 container_part_opl = 0;
U64 leaf_part_first = 0;
for(U64 off = 0; off+1 < name.size; off += 1)
{
if(name.str[off] == '.')
{
container_part_opl = off;
leaf_part_first = off+1;
}
else if(name.str[off] == ':' && name.str[off+1] == ':')
{
container_part_opl = off;
leaf_part_first = off+2;
}
}
//- rjf: unpack container/leaf
String8 container_part_of_name = str8_prefix(name, container_part_opl);
String8 leaf_part_of_name = str8_skip(name, leaf_part_first);
//- rjf: produce the fully qualified name, sanitize the container part (convert all `.`s to `::`s)
String8 fully_qualified_name = name;
if(container_part_of_name.size != 0)
{
// rjf: gather parts of container
String8List parts = {0};
for(U64 off = 0, next_off = 0; off < container_part_of_name.size; off = next_off)
{
// rjf: find the next separator
U64 next_separator_pos = container_part_of_name.size;
U64 next_separator_size = 0;
{
S64 template_nest_depth = 0;
for(U64 off2 = off; off2+1 < container_part_of_name.size; off2 += 1)
{
if(template_nest_depth == 0 && container_part_of_name.str[off2] == '.')
{
next_separator_pos = off2;
next_separator_size = 1;
break;
}
else if(template_nest_depth == 0 && container_part_of_name.str[off2] == ':' && container_part_of_name.str[off2+1] == ':')
{
next_separator_pos = off2;
next_separator_size = 2;
break;
}
else if(container_part_of_name.str[off2] == '<')
{
template_nest_depth += 1;
}
else if(container_part_of_name.str[off2] == '>')
{
template_nest_depth -= 1;
template_nest_depth = ClampBot(0, template_nest_depth);
}
}
}
// rjf: gather portions of container
str8_list_push(scratch.arena, &parts, str8_substr(container_part_of_name, r1u64(off, next_separator_pos)));
next_off = next_separator_pos + next_separator_size;
}
// rjf: add leaf & join
str8_list_push(scratch.arena, &parts, leaf_part_of_name);
StringJoin join = {.sep = str8_lit("::")};
fully_qualified_name = str8_list_join(scratch.arena, &parts, &join);
}
//- rjf: get all loaded keys
DI_KeyArray dbgi_keys = di_push_all_loaded_keys(scratch.arena);
//- rjf: take cancellation signal
B32 cancelled = 0;
if(lane_idx() == 0)
{
cancelled = ins_atomic_u32_eval(cancel_signal);
}
lane_sync_u64(&cancelled, 0);
//- rjf: take cancellation signal
B32 cancelled = 0;
if(lane_idx() == 0)
{
cancelled = ins_atomic_u32_eval(cancel_signal);
}
lane_sync_u64(&cancelled, 0);
//- rjf: wide search across all debug infos
DI_Match *lane_matches = 0;
if(!cancelled)
{
DI_Match *lane_matches = 0;
if(!cancelled)
{
if(lane_idx() == 0)
{
lane_matches = push_array(scratch.arena, DI_Match, lane_count());
@@ -1577,44 +1648,94 @@ di_match_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out, U64 *g
RDI_Parsed *rdi = di_rdi_from_key(access, dbgi_key, 0, 0);
for EachElement(name_map_kind_idx, name_map_kinds)
{
// rjf: unpack name map
RDI_NameMap *name_map = rdi_element_from_name_idx(rdi, NameMaps, name_map_kinds[name_map_kind_idx]);
RDI_ParsedNameMap parsed_name_map = {0};
rdi_parsed_from_name_map(rdi, name_map, &parsed_name_map);
RDI_NameMapNode *map_node = rdi_name_map_lookup(rdi, &parsed_name_map, name.str, name.size);
U32 num = 0;
U32 *run = rdi_matches_from_map_node(rdi, map_node, &num);
if(num != 0)
{
U64 run_idx = (num-1) - Min(index, num-1);
// rjf: find matches for (possibly leaf) name
RDI_NameMapNode *map_node = rdi_name_map_lookup(rdi, &parsed_name_map, leaf_part_of_name.str, leaf_part_of_name.size);
U32 matches_count = 0;
U32 *matches = rdi_matches_from_map_node(rdi, map_node, &matches_count);
// rjf: do we have a container? -> filter matches according to container string also
if(container_part_of_name.size != 0)
{
U32 *filtered_matches = push_array(scratch.arena, U32, matches_count);
U32 filtered_matches_count = 0;
for EachIndex(match_idx, matches_count)
{
Temp scratch = scratch_begin(0, 0);
String8 match_fully_qualified_name = {0};
switch(name_map_section_kinds[name_map_kind_idx])
{
default:{}break;
case RDI_SectionKind_GlobalVariables:
case RDI_SectionKind_ThreadVariables:
case RDI_SectionKind_Constants:
case RDI_SectionKind_Procedures:
{
RDI_Symbol *symbol = (RDI_Symbol *)rdi_section_raw_element_from_kind_idx(rdi, name_map_section_kinds[name_map_kind_idx], matches[match_idx]);
String8 match_fully_qualified_name = fully_qualified_str8_from_rdi_symbol(scratch.arena, rdi, symbol);
if(str8_match(match_fully_qualified_name, fully_qualified_name, 0))
{
filtered_matches[filtered_matches_count] = matches[match_idx];
filtered_matches_count += 1;
}
}break;
case RDI_SectionKind_TypeNodes:
{
RDI_TypeNode *type_node = rdi_element_from_name_idx(rdi, TypeNodes, match_idx);
if(RDI_TypeKind_FirstUserDefined <= type_node->kind && type_node->kind <= RDI_TypeKind_LastUserDefined)
{
String8 match_fully_qualified_name = fully_qualified_str8_from_rdi_type(scratch.arena, rdi, type_node);
if(str8_match(match_fully_qualified_name, fully_qualified_name, 0))
{
filtered_matches[filtered_matches_count] = matches[match_idx];
filtered_matches_count += 1;
}
}
}break;
}
scratch_end(scratch);
}
matches = filtered_matches;
matches_count = filtered_matches_count;
}
// rjf: given matches, pick selected & return as result
if(matches_count != 0)
{
U64 selected_match_idx = (matches_count-1) - Min(index, matches_count-1);
lane_matches[lane_idx()].key = dbgi_key;
lane_matches[lane_idx()].section_kind = name_map_section_kinds[name_map_kind_idx];
lane_matches[lane_idx()].idx = run[run_idx];
lane_matches[lane_idx()].idx = matches[selected_match_idx];
}
}
}
access_close(access);
}
}
}
lane_sync();
}
}
lane_sync();
//- rjf: pick match
DI_Match match = {0};
if(lane_matches != 0)
{
DI_Match match = {0};
if(lane_matches != 0)
{
for EachIndex(idx, lane_count())
{
if(lane_matches[idx].idx != 0)
{
match = lane_matches[idx];
if(di_key_match(di_key_zero(), preferred_key) || di_key_match(match.key, preferred_key))
{
break;
}
match = lane_matches[idx];
if(di_key_match(di_key_zero(), preferred_key) || di_key_match(match.key, preferred_key))
{
break;
}
}
}
}
}
//- rjf: package as artifact
AC_Artifact artifact = {0};
{
@@ -1640,12 +1761,12 @@ di_match_from_string(String8 string, U64 index, DI_Key preferred_dbgi_key, U64 e
{
String8List key_parts = {0};
str8_list_push(scratch.arena, &key_parts, str8_struct(&index));
str8_list_push(scratch.arena, &key_parts, str8_struct(&preferred_dbgi_key));
str8_list_push(scratch.arena, &key_parts, str8_struct(&string.size));
str8_list_push(scratch.arena, &key_parts, str8_struct(&preferred_dbgi_key));
str8_list_push(scratch.arena, &key_parts, str8_struct(&string.size));
str8_list_push(scratch.arena, &key_parts, string);
String8 key = str8_list_join(scratch.arena, &key_parts, 0);
U64 dbgi_count = di_load_count();
B32 wide = (dbgi_count > 256);
String8 key = str8_list_join(scratch.arena, &key_parts, 0);
U64 dbgi_count = di_load_count();
B32 wide = (dbgi_count > 256);
AC_Artifact artifact = ac_artifact_from_key(access, key, di_match_artifact_create, 0, endt_us, .flags = wide ? AC_Flag_Wide : 0, .gen = di_load_gen(), .evict_threshold_us = wide ? 20000000 : 10000000);
result.key.u64[0] = artifact.u64[0];
result.key.u64[1] = artifact.u64[1];