bucket active conversion tasks by priority; adjust thread cap for bg conversion processes, don't flood cores with work while target is doing things

src/config/config.c

@@ -0,0 +1,62 @@
+// Copyright (c) Epic Games Tools
+// Licensed under the MIT license (https://opensource.org/license/mit/)
+
+////////////////////////////////
+//~ rjf: ID Functions
+
+internal void cfg_id_list_push(Arena *arena, CFG_IDList *list, CFG_ID id);
+internal CFG_IDList cfg_id_list_copy(Arena *arena, CFG_IDList *src);
+
+////////////////////////////////
+//~ rjf: Node Pointer Data Structure Functions
+
+internal void cfg_node_ptr_list_push(Arena *arena, CFG_NodePtrList *list, CFG_Node *node);
+internal void cfg_node_ptr_list_push_front(Arena *arena, CFG_NodePtrList *list, CFG_Node *node);
+#define cfg_node_ptr_list_first(list) ((list)->count ? (list)->first->v : &cfg_nil_node)
+#define cfg_node_ptr_list_last(list)  ((list)->count ? (list)->last->v  : &cfg_nil_node)
+internal CFG_NodePtrArray cfg_node_ptr_array_from_list(Arena *arena, CFG_NodePtrList *list);
+
+////////////////////////////////
+//~ rjf: Config Reading Functions
+
+//- rjf: context selection
+internal void cfg_ctx_select(CFG_Ctx *ctx);
+
+//- rjf: tree navigations
+internal CFG_Node *cfg_node_from_id(CFG_ID id);
+internal CFG_Node *cfg_node_child_from_string(CFG_Node *parent, String8 string);
+internal CFG_Node *cfg_node_child_from_string_or_parent(CFG_Node *parent, String8 string);
+internal CFG_NodePtrList cfg_node_child_list_from_string(Arena *arena, CFG_Node *parent, String8 string);
+internal CFG_NodePtrList cfg_node_top_level_list_from_string(Arena *arena, String8 string);
+internal CFG_NodeRec cfg_node_rec__depth_first(CFG_Node *root, CFG_Node *node);
+
+//- rjf: serialization
+internal String8 cfg_string_from_tree(Arena *arena, String8 root_path, CFG_Node *root);
+
+////////////////////////////////
+//~ rjf: Config Writing Functions
+
+//- rjf: state creation / destroying
+internal CFG_State *cfg_state_alloc(void);
+internal void cfg_state_release(CFG_State *state);
+
+//- rjf: state -> ctx
+internal CFG_Ctx *cfg_state_ctx(CFG_State *state);
+
+//- rjf: tree building
+internal CFG_Node *cfg_node_alloc(CFG_State *state);
+internal void cfg_node_release(CFG_State *state, CFG_Node *node);
+internal void cfg_node_release_all_children(CFG_State *state, CFG_Node *node);
+internal CFG_Node *cfg_node_new(CFG_State *state, CFG_Node *parent, String8 string);
+internal CFG_Node *cfg_node_newf(CFG_State *state, CFG_Node *parent, char *fmt, ...);
+internal CFG_Node *cfg_node_new_replace(CFG_State *state, CFG_Node *parent, String8 string);
+internal CFG_Node *cfg_node_new_replacef(CFG_State *state, CFG_Node *parent, char *fmt, ...);
+internal CFG_Node *cfg_node_deep_copy(CFG_State *state, CFG_Node *src_root);
+internal void cfg_node_equip_string(CFG_State *state, CFG_Node *node, String8 string);
+internal void cfg_node_equip_stringf(CFG_State *state, CFG_Node *node, char *fmt, ...);
+internal void cfg_node_insert_child(CFG_State *state, CFG_Node *parent, CFG_Node *prev_child, CFG_Node *new_child);
+internal void cfg_node_unhook(CFG_State *state, CFG_Node *parent, CFG_Node *child);
+internal CFG_Node *cfg_node_child_from_string_or_alloc(CFG_State *state, CFG_Node *parent, String8 string);
+
+//- rjf: deserialization
+internal CFG_NodePtrList cfg_node_ptr_list_from_string(Arena *arena, String8 root_path, String8 string);

src/config/config.h

@@ -0,0 +1,178 @@
+// Copyright (c) Epic Games Tools
+// Licensed under the MIT license (https://opensource.org/license/mit/)
+
+#ifndef CONFIG_H
+#define CONFIG_H
+
+////////////////////////////////
+//~ rjf: IDs
+
+typedef U64 CFG_ID;
+
+typedef struct CFG_IDNode CFG_IDNode;
+struct CFG_IDNode
+{
+  CFG_IDNode *next;
+  CFG_ID v;
+};
+
+typedef struct CFG_IDList CFG_IDList;
+struct CFG_IDList
+{
+  CFG_IDNode *first;
+  CFG_IDNode *last;
+  U64 count;
+};
+
+////////////////////////////////
+//~ rjf: Tree Types
+
+typedef struct CFG_Node CFG_Node;
+struct CFG_Node
+{
+  CFG_Node *first;
+  CFG_Node *last;
+  CFG_Node *next;
+  CFG_Node *prev;
+  CFG_Node *parent;
+  CFG_ID id;
+  String8 string;
+};
+
+typedef struct CFG_NodePtrNode CFG_NodePtrNode;
+struct CFG_NodePtrNode
+{
+  CFG_NodePtrNode *next;
+  CFG_NodePtrNode *prev;
+  CFG_Node *v;
+};
+
+typedef struct CFG_NodePtrSlot CFG_NodePtrSlot;
+struct CFG_NodePtrSlot
+{
+  CFG_NodePtrNode *first;
+  CFG_NodePtrNode *last;
+};
+
+typedef struct CFG_NodePtrList CFG_NodePtrList;
+struct CFG_NodePtrList
+{
+  CFG_NodePtrNode *first;
+  CFG_NodePtrNode *last;
+  U64 count;
+};
+
+typedef struct CFG_NodePtrArray CFG_NodePtrArray;
+struct CFG_NodePtrArray
+{
+  CFG_Node **v;
+  U64 count;
+};
+
+typedef struct CFG_NodeRec CFG_NodeRec;
+struct CFG_NodeRec
+{
+  CFG_Node *next;
+  S32 push_count;
+  S32 pop_count;
+};
+
+////////////////////////////////
+//~ rjf: Config State Bundles
+
+typedef struct CFG_Ctx CFG_Ctx;
+struct CFG_Ctx
+{
+  CFG_Node *root;
+  U64 id_slots_count;
+  CFG_NodePtrSlot *id_slots;
+  U64 change_gen;
+  CFG_ID last_accessed_id;
+  CFG_Node *last_accessed;
+};
+
+typedef struct CFG_State CFG_State;
+struct CFG_State
+{
+  Arena *arena;
+  CFG_Node *free;
+  CFG_NodePtrNode *free_id_node;
+  U64 id_gen;
+  CFG_Ctx ctx;
+};
+
+////////////////////////////////
+//~ rjf: Globals
+
+read_only global CFG_Node cfg_nil_node =
+{
+  &cfg_nil_node,
+  &cfg_nil_node,
+  &cfg_nil_node,
+  &cfg_nil_node,
+  &cfg_nil_node,
+};
+
+thread_static CFG_Ctx *cfg_ctx = 0;
+
+////////////////////////////////
+//~ rjf: ID Functions
+
+internal void cfg_id_list_push(Arena *arena, CFG_IDList *list, CFG_ID id);
+internal CFG_IDList cfg_id_list_copy(Arena *arena, CFG_IDList *src);
+
+////////////////////////////////
+//~ rjf: Node Pointer Data Structure Functions
+
+internal void cfg_node_ptr_list_push(Arena *arena, CFG_NodePtrList *list, CFG_Node *node);
+internal void cfg_node_ptr_list_push_front(Arena *arena, CFG_NodePtrList *list, CFG_Node *node);
+#define cfg_node_ptr_list_first(list) ((list)->count ? (list)->first->v : &cfg_nil_node)
+#define cfg_node_ptr_list_last(list)  ((list)->count ? (list)->last->v  : &cfg_nil_node)
+internal CFG_NodePtrArray cfg_node_ptr_array_from_list(Arena *arena, CFG_NodePtrList *list);
+
+////////////////////////////////
+//~ rjf: Config Reading Functions
+
+//- rjf: context selection
+internal void cfg_ctx_select(CFG_Ctx *ctx);
+
+//- rjf: tree navigations
+internal CFG_Node *cfg_node_from_id(CFG_ID id);
+internal CFG_Node *cfg_node_child_from_string(CFG_Node *parent, String8 string);
+internal CFG_Node *cfg_node_child_from_string_or_parent(CFG_Node *parent, String8 string);
+internal CFG_NodePtrList cfg_node_child_list_from_string(Arena *arena, CFG_Node *parent, String8 string);
+internal CFG_NodePtrList cfg_node_top_level_list_from_string(Arena *arena, String8 string);
+internal CFG_NodeRec cfg_node_rec__depth_first(CFG_Node *root, CFG_Node *node);
+
+//- rjf: serialization
+internal String8 cfg_string_from_tree(Arena *arena, String8 root_path, CFG_Node *root);
+
+////////////////////////////////
+//~ rjf: Config Writing Functions
+
+//- rjf: state creation / destroying
+internal CFG_State *cfg_state_alloc(void);
+internal void cfg_state_release(CFG_State *state);
+
+//- rjf: state -> ctx
+internal CFG_Ctx *cfg_state_ctx(CFG_State *state);
+
+//- rjf: tree building
+internal CFG_Node *cfg_node_alloc(CFG_State *state);
+internal void cfg_node_release(CFG_State *state, CFG_Node *node);
+internal void cfg_node_release_all_children(CFG_State *state, CFG_Node *node);
+internal CFG_Node *cfg_node_new(CFG_State *state, CFG_Node *parent, String8 string);
+internal CFG_Node *cfg_node_newf(CFG_State *state, CFG_Node *parent, char *fmt, ...);
+internal CFG_Node *cfg_node_new_replace(CFG_State *state, CFG_Node *parent, String8 string);
+internal CFG_Node *cfg_node_new_replacef(CFG_State *state, CFG_Node *parent, char *fmt, ...);
+internal CFG_Node *cfg_node_deep_copy(CFG_State *state, CFG_Node *src_root);
+internal void cfg_node_equip_string(CFG_State *state, CFG_Node *node, String8 string);
+internal void cfg_node_equip_stringf(CFG_State *state, CFG_Node *node, char *fmt, ...);
+internal void cfg_node_insert_child(CFG_State *state, CFG_Node *parent, CFG_Node *prev_child, CFG_Node *new_child);
+internal void cfg_node_unhook(CFG_State *state, CFG_Node *parent, CFG_Node *child);
+internal CFG_Node *cfg_node_child_from_string_or_alloc(CFG_State *state, CFG_Node *parent, String8 string);
+
+//- rjf: deserialization
+internal CFG_NodePtrList cfg_node_ptr_list_from_string(Arena *arena, String8 root_path, String8 string);
+
+#endif // CONFIG_H

src/dbg_info/dbg_info.c

@@ -553,8 +553,8 @@ di_async_tick(void)
     ////////////////////////////
     //- rjf: pop all requests, high priority first
     //
-    DI_RequestNode *first_req = 0;
+    DI_RequestNode *first_req[2] = {0};
-    DI_RequestNode *last_req = 0;
+    DI_RequestNode *last_req[2] = {0};
     for EachElement(idx, di_shared->req_batches)
     {
       DI_RequestBatch *b = &di_shared->req_batches[idx];
@@ -564,7 +564,7 @@ di_async_tick(void)
         {
           DI_RequestNode *n_copy = push_array(scratch.arena, DI_RequestNode, 1);
           MemoryCopyStruct(&n_copy->v, &n->v);
-          SLLQueuePush(first_req, last_req, n_copy);
+          SLLQueuePush(first_req[idx], last_req[idx], n_copy);
         }
         arena_clear(b->arena);
         b->first = b->last = 0;
@@ -592,7 +592,9 @@ di_async_tick(void)
     ////////////////////////////
     //- rjf: generate load tasks for all unique requests
     //
-    for EachNode(n, DI_RequestNode, first_req)
+    for EachElement(priority_idx, first_req)
+    {
+      for EachNode(n, DI_RequestNode, first_req[priority_idx])
       {
         // rjf: unpack request
         DI_Key key = n->v.key;
@@ -630,15 +632,18 @@ di_async_tick(void)
             t = push_array_no_zero(di_shared->arena, DI_LoadTask, 1);
           }
           MemoryZeroStruct(t);
-        DLLPushBack(di_shared->first_load_task, di_shared->last_load_task, t);
+          DLLPushBack(di_shared->first_load_task[priority_idx], di_shared->last_load_task[priority_idx], t);
           t->key = key;
         }
       }
+    }
     
     ////////////////////////////
     //- rjf: update tasks: configure, launch if we can, & retire if we can
     //
-    for(DI_LoadTask *t = di_shared->first_load_task, *next = 0; t != 0; t = next)
+    for EachElement(priority_idx, di_shared->first_load_task)
+    {
+      for(DI_LoadTask *t = di_shared->first_load_task[priority_idx], *next = 0; t != 0; t = next)
       {
         next = t->next;
         
@@ -724,6 +729,10 @@ di_async_tick(void)
         {
           U64 thread_count = 1;
           U64 max_thread_count = os_get_system_info()->logical_processor_count;
+          if(priority_idx > 0)
+          {
+            max_thread_count = Max(1, max_thread_count/2);
+          }
           {
             if(0){}
             else if(og_size <= MB(4))   {thread_count = 1;}
@@ -739,7 +748,7 @@ di_async_tick(void)
         //- rjf: determine if there are threads available
         B32 threads_available = 0;
         {
-        U64 max_threads = os_get_system_info()->logical_processor_count*2;
+          U64 max_threads = os_get_system_info()->logical_processor_count;
           U64 current_threads = di_shared->conversion_thread_count;
           U64 needed_threads = (current_threads + t->thread_count);
           threads_available = (max_threads >= needed_threads);
@@ -837,7 +846,7 @@ di_async_tick(void)
             n->v.kind = DI_EventKind_ConversionEnded;
             n->v.string = str8_copy(di_shared->event_arena, rdi_path);
           }
-        DLLRemove(di_shared->first_load_task, di_shared->last_load_task, t);
+          DLLRemove(di_shared->first_load_task[priority_idx], di_shared->last_load_task[priority_idx], t);
           SLLStackPush(di_shared->free_load_task, t);
           ParseTaskNode *n = push_array(scratch.arena, ParseTaskNode, 1);
           n->v.key = key;
@@ -846,6 +855,7 @@ di_async_tick(void)
           parse_tasks_count += 1;
         }
       }
+    }
     
     ////////////////////////////
     //- rjf: join all parse tasks

src/dbg_info/dbg_info.h

@@ -267,8 +267,8 @@ struct DI_Shared
   DI_RequestBatch req_batches[2]; // [0] -> high priority, [1] -> low priority
   
   // rjf: conversion tasks
-  DI_LoadTask *first_load_task;
+  DI_LoadTask *first_load_task[2];
-  DI_LoadTask *last_load_task;
+  DI_LoadTask *last_load_task[2];
   DI_LoadTask *free_load_task;
   U64 conversion_process_count;
   U64 conversion_thread_count;

src/raddbg/raddbg_main.c

@@ -148,13 +148,13 @@
 // [ ] step-out-of-loop
 //
 //- late-conversion performance improvements
-// [ ] investigate wide-conversion performance
+// [x] investigate wide-conversion performance
-//  [ ] oversubscribing cores?
+//  [x] oversubscribing cores?
-//  [ ] conversion crashes?
+//  [x] conversion crashes?
 // [ ] live++ investigations - ctrl+alt+f11 in UE?
 //
 //- memory usage improvements
-// [ ] "root" concept in hash store, which buckets keys & allows usage code to
+// [x] "root" concept in hash store, which buckets keys & allows usage code to
 //     jettison a collection of keys in retained mode fashion
 //
 //- short-to-medium term future features