correctly interpret conversion completions; single high priority re-loop bit for async threads

src/artifact_cache/artifact_cache.c

@@ -166,6 +166,10 @@ ac_artifact_from_key_(Access *access, String8 key, AC_ArtifactParams *params, U6
         }
         cond_var_broadcast(async_tick_start_cond_var);
         ins_atomic_u32_eval_assign(&async_loop_again, 1);
+        if(params->flags & AC_Flag_HighPriority)
+        {
+          ins_atomic_u32_eval_assign(&async_loop_again_high_priority, 1);
+        }
       }
       
       // rjf: get value from node, if possible
@@ -342,18 +346,15 @@ ac_async_tick(void)
         // rjf: any new higher priority tasks? -> cancel
         if(lane_idx() == 0)
         {
-          if(task_idx == 1 && idx != 0) MutexScope(ac_shared->req_batches[0].mutex)
+          if(task_idx == 1 && idx != 0 && ins_atomic_u32_eval(&async_loop_again_high_priority))
           {
-            if(ac_shared->req_batches[0].wide_count != 0 || ac_shared->req_batches[0].thin_count != 0)
+            ins_atomic_u64_eval_assign(cancelled_ptr, 1);
-            {
-              ins_atomic_u64_eval_assign(cancelled_ptr, 1);
-            }
           }
         }
         lane_sync();
         
         // rjf: cancelled? -> exit
-        if(ins_atomic_u64_eval(cancelled_ptr))
+        if(ins_atomic_u32_eval(cancelled_ptr))
         {
           break;
         }
@@ -441,12 +442,10 @@ ac_async_tick(void)
       for(;;)
       {
         // rjf: any new higher priority tasks? -> cancel
-        if(task_idx == 1 && ins_atomic_u64_eval(req_take_counter_ptr) >= task->thin_count/2) MutexScope(ac_shared->req_batches[0].mutex)
+        if(task_idx == 1 && ins_atomic_u64_eval(req_take_counter_ptr) >= task->thin_count/2 &&
+           ins_atomic_u32_eval(&async_loop_again_high_priority))
         {
-          if(ac_shared->req_batches[0].wide_count != 0 || ac_shared->req_batches[0].thin_count != 0)
+          ins_atomic_u64_eval_assign(cancelled_ptr, 1);
-          {
-            ins_atomic_u64_eval_assign(cancelled_ptr, 1);
-          }
         }
         
         // rjf: cancelled? -> exit

src/base/base_entry_point.c

@@ -7,6 +7,7 @@ global CondVar async_tick_start_cond_var = {0};
 global Mutex async_tick_start_mutex = {0};
 global Mutex async_tick_stop_mutex = {0};
 global B32 async_loop_again = 0;
+global B32 async_loop_again_high_priority = 0;
 global B32 global_async_exit = 0;
 thread_static B32 is_async_thread = 0;
 

src/ctrl/ctrl_core.c

@@ -3658,7 +3658,6 @@ ctrl_thread__module_open(CTRL_Handle process, CTRL_Handle module, Rng1U64 vaddr_
     str8_list_pushf(scratch.arena, &dbg_path_candidates, "%S.pdb", path);
     str8_list_pushf(scratch.arena, &dbg_path_candidates, "%S.rdi", str8_chop_last_dot(path));
     str8_list_pushf(scratch.arena, &dbg_path_candidates, "%S.rdi", path);
-    str8_list_push(scratch.arena, &dbg_path_candidates, path);
     for(String8Node *n = dbg_path_candidates.first; n != 0; n = n->next)
     {
       String8 candidate_path = n->string;
@@ -4360,7 +4359,7 @@ ctrl_thread__eval_scope_begin(Arena *arena, CTRL_UserBreakpointList *user_bps, C
           }
           
           //- rjf: if this RDI is necessary, but we do not have it => wait for it forever
-          if(rdi == &rdi_parsed_nil && rdi_is_necessary)
+          if(rdi == &rdi_parsed_nil && rdi_is_necessary) ProfScope("RDI is necessary -> wait")
           {
             rdi = di2_rdi_from_key(scope->access, dbgi_key, 1, max_U64);
           }

src/dbg_info/dbg_info2.c

@@ -73,16 +73,29 @@ di2_init(CmdLine *cmdline)
     has_parent = 0;
     signal_pid = os_get_process_info()->pid;
   }
+  U64 signal_code = 0;
+  String8 signal_code_string = cmd_line_string(cmdline, str8_lit("signal_code"));
+  try_u64_from_str8_c_rules(signal_code_string, &signal_code);
+  di2_shared->conversion_completion_code = signal_code;
+  di2_shared->conversion_completion_lock_semaphore_name = str8f(arena, "conversion_completion_lock_pid_%I64u", signal_pid);
   di2_shared->conversion_completion_signal_semaphore_name = str8f(arena, "conversion_completion_signal_pid_%I64u", signal_pid);
+  di2_shared->conversion_completion_shared_memory_name = str8f(arena, "conversion_completion_shared_memory_pid_%I64u", signal_pid);
   if(has_parent)
   {
+    di2_shared->conversion_completion_lock_semaphore = semaphore_open(di2_shared->conversion_completion_lock_semaphore_name);
     di2_shared->conversion_completion_signal_semaphore = semaphore_open(di2_shared->conversion_completion_signal_semaphore_name);
+    di2_shared->conversion_completion_shared_memory = os_shared_memory_open(di2_shared->conversion_completion_shared_memory_name);
   }
   else
   {
+    di2_shared->conversion_completion_lock_semaphore = semaphore_alloc(1, 1, di2_shared->conversion_completion_lock_semaphore_name);
     di2_shared->conversion_completion_signal_semaphore = semaphore_alloc(0, 65536, di2_shared->conversion_completion_signal_semaphore_name);
+    di2_shared->conversion_completion_shared_memory = os_shared_memory_alloc(KB(4), di2_shared->conversion_completion_shared_memory_name);
     di2_shared->conversion_completion_signal_receiver_thread = thread_launch(di2_conversion_completion_signal_receiver_thread_entry_point, 0);
   }
+  di2_shared->conversion_completion_shared_memory_base = (U64 *)os_shared_memory_view_open(di2_shared->conversion_completion_shared_memory, r1u64(0, KB(4)));
+  di2_shared->completion_mutex = mutex_alloc();
+  di2_shared->completion_arena = arena_alloc();
 }
 
 ////////////////////////////////
@@ -456,6 +469,7 @@ di2_rdi_from_key(Access *access, DI2_Key key, B32 high_priority, U64 endt_us)
         }
         cond_var_broadcast(async_tick_start_cond_var);
         ins_atomic_u32_eval_assign(&async_loop_again, 1);
+        ins_atomic_u32_eval_assign(&async_loop_again_high_priority, 1);
       }
       
       // rjf: found current results, or out-of-time? abort
@@ -523,6 +537,23 @@ di2_async_tick(void)
       }
     }
     
+    ////////////////////////////
+    //- rjf: gather all completions
+    //
+    DI2_LoadCompletion *first_completion = 0;
+    DI2_LoadCompletion *last_completion = 0;
+    MutexScope(di2_shared->completion_mutex)
+    {
+      for EachNode(c, DI2_LoadCompletion, di2_shared->first_completion)
+      {
+        DI2_LoadCompletion *dst_c = push_array(scratch.arena, DI2_LoadCompletion, 1);
+        SLLQueuePush(first_completion, last_completion, dst_c);
+        dst_c->code = c->code;
+      }
+      arena_clear(di2_shared->completion_arena);
+      di2_shared->first_completion = di2_shared->last_completion = 0;
+    }
+    
     ////////////////////////////
     //- rjf: generate load tasks for all unique requests
     //
@@ -699,7 +730,9 @@ di2_async_tick(void)
         str8_list_pushf(scratch.arena, &params.cmd_line, "--out:%S", rdi_path);
         str8_list_pushf(scratch.arena, &params.cmd_line, "--thread_count:%I64u", t->thread_count);
         str8_list_pushf(scratch.arena, &params.cmd_line, "--signal_pid:%I64u", (U64)os_get_process_info()->pid);
+        str8_list_pushf(scratch.arena, &params.cmd_line, "--signal_code:%I64u", (U64)t);
         str8_list_pushf(scratch.arena, &params.cmd_line, "%S", og_path);
+        ProfMsg("launch creation for %.*s", str8_varg(rdi_path));
         t->process = os_process_launch(&params);
         t->status = DI2_LoadTaskStatus_Active;
         di2_shared->conversion_process_count += 1;
@@ -709,11 +742,27 @@ di2_async_tick(void)
       //- rjf: if active & process has completed, mark as done
       {
         U64 exit_code = 0;
-        if(t->status == DI2_LoadTaskStatus_Active && os_process_join(t->process, 0, &exit_code))
+        if(t->status == DI2_LoadTaskStatus_Active)
         {
-          t->status = DI2_LoadTaskStatus_Done;
+          B32 task_is_done = 0;
-          di2_shared->conversion_process_count -= 1;
+          for(DI2_LoadCompletion *c = first_completion; c != 0; c = c->next)
-          di2_shared->conversion_thread_count -= t->thread_count;
+          {
+            if(c->code == (U64)t)
+            {
+              task_is_done = 1;
+              break;
+            }
+          }
+          if(!task_is_done)
+          {
+            task_is_done = os_process_join(t->process, 0, 0);
+          }
+          if(task_is_done)
+          {
+            t->status = DI2_LoadTaskStatus_Done;
+            di2_shared->conversion_process_count -= 1;
+            di2_shared->conversion_thread_count -= t->thread_count;
+          }
         }
       }
       
@@ -823,6 +872,7 @@ di2_async_tick(void)
       
       //- rjf: commit parsed info to cache
       {
+        ProfMsg("commit %.*s", str8_varg(rdi_path));
         U64 hash = u64_hash_from_str8(str8_struct(&key));
         U64 slot_idx = hash%di2_shared->slots_count;
         DI2_Slot *slot = &di2_shared->slots[slot_idx];
@@ -878,6 +928,9 @@ di2_async_tick(void)
 internal void
 di2_signal_completion(void)
 {
+  semaphore_take(di2_shared->conversion_completion_lock_semaphore, max_U64);
+  di2_shared->conversion_completion_shared_memory_base[0] = di2_shared->conversion_completion_code;
+  semaphore_drop(di2_shared->conversion_completion_lock_semaphore);
   semaphore_drop(di2_shared->conversion_completion_signal_semaphore);
 }
 
@@ -889,8 +942,25 @@ di2_conversion_completion_signal_receiver_thread_entry_point(void *p)
   {
     if(semaphore_take(di2_shared->conversion_completion_signal_semaphore, max_U64))
     {
-      cond_var_broadcast(async_tick_start_cond_var);
+      // rjf: get the next retired code
+      U64 retired_code = 0;
+      semaphore_take(di2_shared->conversion_completion_lock_semaphore, max_U64);
+      retired_code = di2_shared->conversion_completion_shared_memory_base[0];
+      semaphore_drop(di2_shared->conversion_completion_lock_semaphore);
+      
+      // rjf: push completion record
+      MutexScope(di2_shared->completion_mutex)
+      {
+        DI2_LoadCompletion *c = push_array(di2_shared->completion_arena, DI2_LoadCompletion, 1);
+        SLLQueuePush(di2_shared->first_completion, di2_shared->last_completion, c);
+        c->code = retired_code;
+      }
+      
+      // rjf: signal async system to resume
+      ProfMsg("signal conversion completion");
       ins_atomic_u32_eval_assign(&async_loop_again, 1);
+      ins_atomic_u32_eval_assign(&async_loop_again_high_priority, 1);
+      cond_var_broadcast(async_tick_start_cond_var);
     }
   }
 }
@@ -1312,6 +1382,7 @@ di2_search_item_array_from_target_query(Access *access, RDI_SectionKind target,
 internal AC_Artifact
 di2_match_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out)
 {
+  ProfBeginFunction();
   Temp scratch = scratch_begin(0, 0);
   
   //- rjf: unpack key
@@ -1403,6 +1474,7 @@ di2_match_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out)
   
   lane_sync();
   scratch_end(scratch);
+  ProfEnd();
   return artifact;
 }
 

src/dbg_info/dbg_info2.h

@@ -148,6 +148,13 @@ struct DI2_LoadTask
   OS_Handle process;
 };
 
+typedef struct DI2_LoadCompletion DI2_LoadCompletion;
+struct DI2_LoadCompletion
+{
+  DI2_LoadCompletion *next;
+  U64 code;
+};
+
 ////////////////////////////////
 //~ rjf: Search Types
 
@@ -232,9 +239,21 @@ struct DI2_Shared
   U64 conversion_thread_count;
   
   // rjf: conversion completion receiving thread
+  U64 conversion_completion_code;
+  String8 conversion_completion_lock_semaphore_name;
   String8 conversion_completion_signal_semaphore_name;
+  String8 conversion_completion_shared_memory_name;
+  Semaphore conversion_completion_lock_semaphore;
   Semaphore conversion_completion_signal_semaphore;
+  OS_Handle conversion_completion_shared_memory;
+  U64 *conversion_completion_shared_memory_base;
   Thread conversion_completion_signal_receiver_thread;
+  
+  // rjf: completion batch
+  Mutex completion_mutex;
+  Arena *completion_arena;
+  DI2_LoadCompletion *first_completion;
+  DI2_LoadCompletion *last_completion;
 };
 
 ////////////////////////////////

src/raddbg/raddbg_main.c

@@ -248,7 +248,6 @@
 #include "radbin/radbin.h"
 #include "regs/regs.h"
 #include "regs/rdi/regs_rdi.h"
-// #include "dbg_info/dbg_info.h"
 #include "dbg_info/dbg_info2.h"
 #include "disasm/disasm.h"
 #include "demon/demon_inc.h"
@@ -296,7 +295,6 @@
 #include "radbin/radbin.c"
 #include "regs/regs.c"
 #include "regs/rdi/regs_rdi.c"
-// #include "dbg_info/dbg_info.c"
 #include "dbg_info/dbg_info2.c"
 #include "disasm/disasm.c"
 #include "demon/demon_inc.c"