further progress on integrating new demon layer

src/ctrl/ctrl_core.c

@@ -933,7 +933,7 @@ ctrl_stored_hash_from_process_vaddr_range(CTRL_MachineID machine_id, DMN_Handle
             {
               result = range_n->hash;
               is_good = 1;
-              is_stale = (range_n->memgen_idx < ctrl_memgen_idx());
+              is_stale = (range_n->mem_gen != dmn_mem_gen());
               goto read_cache__break_all;
             }
           }
@@ -1169,6 +1169,20 @@ ctrl_query_cached_data_from_process_vaddr_range(Arena *arena, CTRL_MachineID mac
     result.data.size = dim_1u64(range);
     result.byte_bad_flags = byte_bad_flags;
     result.byte_changed_flags = byte_changed_flags;
+    if(byte_bad_flags != 0)
+    {
+      for(U64 idx = 0; idx < (dim_1u64(range)+63)/64; idx += 1)
+      {
+        result.any_byte_bad = result.any_byte_bad || !!result.byte_bad_flags[idx];
+      }
+    }
+    if(byte_changed_flags != 0)
+    {
+      for(U64 idx = 0; idx < (dim_1u64(range)+63)/64; idx += 1)
+      {
+        result.any_byte_changed = result.any_byte_changed || !!result.byte_changed_flags[idx];
+      }
+    }
     
     hs_scope_close(scope);
     scratch_end(scratch);
@@ -1199,12 +1213,6 @@ ctrl_process_write(CTRL_MachineID machine_id, DMN_Handle process, Rng1U64 range,
   ProfBeginFunction();
   B32 result = dmn_process_write(process, range, src);
   
-  //- rjf: success -> increment memgen
-  if(result)
-  {
-    ins_atomic_u64_inc_eval(&ctrl_state->memgen_idx);
-  }
-  
   //- rjf: success -> wait for cache updates, for small regions - prefer relatively seamless
   // writes within calling frame's "view" of the memory, at the expense of a small amount of
   // time.
@@ -1335,10 +1343,10 @@ ctrl_query_cached_reg_block_from_thread(Arena *arena, CTRL_MachineID machine_id,
     // rjf: copy from node
     if(node)
     {
-      U64 current_reggen_idx = ctrl_reggen_idx();
+      U64 current_reg_gen = dmn_reg_gen();
-      if(node->reggen_idx != current_reggen_idx && dmn_thread_read_reg_block(thread, node->block))
+      if(node->reg_gen != current_reg_gen && dmn_thread_read_reg_block(thread, node->block))
       {
-        node->reggen_idx = current_reggen_idx;
+        node->reg_gen = current_reg_gen;
       }
       MemoryCopy(result, node->block, reg_block_size);
     }
@@ -1381,7 +1389,6 @@ internal B32
 ctrl_thread_write_reg_block(CTRL_MachineID machine_id, DMN_Handle thread, void *block)
 {
   B32 good = dmn_thread_write_reg_block(thread, block);
-  ins_atomic_u64_inc_eval(&ctrl_state->reggen_idx);
   return good;
 }
 
@@ -1421,7 +1428,7 @@ ctrl_unwind_from_thread(Arena *arena, CTRL_EntityStore *store, CTRL_MachineID ma
         {
           CTRL_ProcessMemorySlice stack_memory_slice = ctrl_query_cached_data_from_process_vaddr_range(scratch.arena, machine_id, process_entity->handle, r1u64(stack_top, stack_top+stack_size), endt_us);
           String8 stack_memory = stack_memory_slice.data;
-          if(stack_memory.size != 0)
+          if(stack_memory.size != 0 && !stack_memory_slice.any_byte_bad)
           {
             stack_memview_good = 1;
             stack_memview.data = stack_memory.str;
@@ -1517,26 +1524,26 @@ ctrl_halt(void)
 ////////////////////////////////
 //~ rjf: Shared Accessor Functions
 
-//- rjf: run indices
+//- rjf: run generation counter
 
 internal U64
-ctrl_run_idx(void)
+ctrl_run_gen(void)
 {
-  U64 result = ins_atomic_u64_eval(&ctrl_state->run_idx);
+  U64 result = dmn_run_gen();
   return result;
 }
 
 internal U64
-ctrl_memgen_idx(void)
+ctrl_mem_gen(void)
 {
-  U64 result = ins_atomic_u64_eval(&ctrl_state->memgen_idx);
+  U64 result = dmn_mem_gen();
   return result;
 }
 
 internal U64
-ctrl_reggen_idx(void)
+ctrl_reg_gen(void)
 {
-  U64 result = ins_atomic_u64_eval(&ctrl_state->reggen_idx);
+  U64 result = dmn_reg_gen();
   return result;
 }
 
@@ -2054,13 +2061,6 @@ ctrl_thread__next_dmn_event(Arena *arena, CTRL_Msg *msg, DMN_RunCtrls *run_ctrls
       {
         dmn_process_write(spoof->process, r1u64(spoof->vaddr, spoof->vaddr+size_of_spoof), &spoof_old_ip_value);
       }
-      
-      // rjf: inc generation counters
-      {
-        ins_atomic_u64_inc_eval(&ctrl_state->run_idx);
-        ins_atomic_u64_inc_eval(&ctrl_state->memgen_idx);
-        ins_atomic_u64_inc_eval(&ctrl_state->reggen_idx);
-      }
     }
   }
   
@@ -3651,7 +3651,7 @@ ctrl_mem_stream_thread__entry_point(void *p)
     
     //- rjf: take task
     B32 got_task = 0;
-    U64 preexisting_memgen_idx = 0;
+    U64 preexisting_mem_gen = 0;
     U128 preexisting_hash = {0};
     Rng1U64 vaddr_range_clamped = {0};
     OS_MutexScopeW(process_stripe->rw_mutex)
@@ -3667,7 +3667,7 @@ ctrl_mem_stream_thread__entry_point(void *p)
             if(MemoryMatchStruct(&range_n->vaddr_range, &vaddr_range) && range_n->zero_terminated == zero_terminated)
             {
               got_task = !ins_atomic_u32_eval_cond_assign(&range_n->is_taken, 1, 0);
-              preexisting_memgen_idx = range_n->memgen_idx;
+              preexisting_mem_gen = range_n->mem_gen;
               preexisting_hash = range_n->hash;
               vaddr_range_clamped = range_n->vaddr_range_clamped;
               goto take_task__break_all;
@@ -3683,8 +3683,8 @@ ctrl_mem_stream_thread__entry_point(void *p)
     Arena *range_arena = 0;
     void *range_base = 0;
     U64 zero_terminated_size = 0;
-    U64 memgen_idx = ctrl_memgen_idx();
+    U64 mem_gen = dmn_mem_gen();
-    if(got_task && memgen_idx != preexisting_memgen_idx)
+    if(got_task && mem_gen != preexisting_mem_gen)
     {
       range_size = dim_1u64(vaddr_range_clamped);
       U64 arena_size = AlignPow2(range_size + ARENA_HEADER_SIZE, os_page_size());
@@ -3746,7 +3746,7 @@ ctrl_mem_stream_thread__entry_point(void *p)
               if(!u128_match(u128_zero(), hash))
               {
                 range_n->hash = hash;
-                range_n->memgen_idx = memgen_idx;
+                range_n->mem_gen = mem_gen;
               }
               ins_atomic_u32_eval_assign(&range_n->is_taken, 0);
               goto commit__break_all;

src/ctrl/ctrl_core.h

@@ -376,7 +376,7 @@ struct CTRL_ProcessMemoryRangeHashNode
   B32 zero_terminated;
   Rng1U64 vaddr_range_clamped;
   U128 hash;
-  U64 memgen_idx;
+  U64 mem_gen;
   U64 last_time_requested_us;
   B32 is_taken;
 };
@@ -429,6 +429,8 @@ struct CTRL_ProcessMemorySlice
   String8 data;
   U64 *byte_bad_flags;
   U64 *byte_changed_flags;
+  B32 any_byte_bad;
+  B32 any_byte_changed;
 };
 
 ////////////////////////////////
@@ -443,7 +445,7 @@ struct CTRL_ThreadRegCacheNode
   DMN_Handle thread;
   U64 block_size;
   void *block;
-  U64 reggen_idx;
+  U64 reg_gen;
 };
 
 typedef struct CTRL_ThreadRegCacheSlot CTRL_ThreadRegCacheSlot;
@@ -483,9 +485,6 @@ struct CTRL_State
 {
   Arena *arena;
   CTRL_WakeupFunctionType *wakeup_hook;
-  U64 run_idx;
-  U64 memgen_idx;
-  U64 reggen_idx;
   
   // rjf: name -> register/alias hash tables for eval
   EVAL_String2NumMap arch_string2reg_tables[Architecture_COUNT];
@@ -672,10 +671,10 @@ internal void ctrl_halt(void);
 ////////////////////////////////
 //~ rjf: Shared Accessor Functions
 
-//- rjf: run indices
+//- rjf: generation counters
-internal U64 ctrl_run_idx(void);
+internal U64 ctrl_run_gen(void);
-internal U64 ctrl_memgen_idx(void);
+internal U64 ctrl_mem_gen(void);
-internal U64 ctrl_reggen_idx(void);
+internal U64 ctrl_reg_gen(void);
 
 //- rjf: name -> register/alias hash tables, for eval
 internal EVAL_String2NumMap *ctrl_string2reg_from_arch(Architecture arch);

src/demon2/demon2_core.h

@@ -165,6 +165,13 @@ internal DMN_Event *dmn_event_list_push(Arena *arena, DMN_EventList *list);
 
 internal void dmn_init(void);
 
+////////////////////////////////
+//~ rjf: @dmn_os_hooks Run/Memory/Register Counters
+
+internal U64 dmn_run_gen(void);
+internal U64 dmn_mem_gen(void);
+internal U64 dmn_reg_gen(void);
+
 ////////////////////////////////
 //~ rjf: @dmn_os_hooks Running/Halting (Implemented Per-OS)
 

src/demon2/win32/demon2_core_win32.c

@@ -343,6 +343,7 @@ dmn_w32_process_write(HANDLE process, Rng1U64 range, void *src)
     ptr += actual_write;
     cursor += actual_write;
   }
+  ins_atomic_u64_inc_eval(&dmn_w32_shared->mem_gen);
   return result;
 }
 
@@ -953,6 +954,7 @@ dmn_w32_thread_write_reg_block(Architecture arch, HANDLE thread, void *reg_block
       //- rjf: bad context -> abort
       if(ctx == 0)
       {
+        DWORD error = GetLastError();
         break;
       }
       
@@ -1043,6 +1045,7 @@ dmn_w32_thread_write_reg_block(Architecture arch, HANDLE thread, void *reg_block
       scratch_end(scratch);
     }break;
   }
+  ins_atomic_u64_inc_eval(&dmn_w32_shared->reg_gen);
   return result;
 }
 
@@ -1104,6 +1107,30 @@ dmn_init(void)
   }
 }
 
+////////////////////////////////
+//~ rjf: @dmn_os_hooks Run/Memory/Register Counters
+
+internal U64
+dmn_run_gen(void)
+{
+  U64 result = ins_atomic_u64_eval(&dmn_w32_shared->run_gen);
+  return result;
+}
+
+internal U64
+dmn_mem_gen(void)
+{
+  U64 result = ins_atomic_u64_eval(&dmn_w32_shared->mem_gen);
+  return result;
+}
+
+internal U64
+dmn_reg_gen(void)
+{
+  U64 result = ins_atomic_u64_eval(&dmn_w32_shared->reg_gen);
+  return result;
+}
+
 ////////////////////////////////
 //~ rjf: @dmn_os_hooks Running/Halting (Implemented Per-OS)
 
@@ -1366,6 +1393,7 @@ dmn_run(Arena *arena, DMN_RunCtrls *ctrls)
             dmn_w32_shared->resume_pid = evt.dwProcessId;
             dmn_w32_shared->resume_tid = evt.dwThreadId;
           }
+          ins_atomic_u64_inc_eval(&dmn_w32_shared->run_gen);
         }
       }
       
@@ -1776,7 +1804,7 @@ dmn_run(Arena *arena, DMN_RunCtrls *ctrls)
               {
                 post_trap_rip = regs_rip_from_arch_block(thread->arch, regs_block);
                 regs_arch_block_write_rip(thread->arch, regs_block, instruction_pointer);
-                dmn_w32_thread_write_reg_block(thread->arch, thread, regs_block);
+                dmn_w32_thread_write_reg_block(thread->arch, thread->handle, regs_block);
               }
               temp_end(temp);
             }

src/demon2/win32/demon2_core_win32.h

@@ -194,6 +194,11 @@ struct DMN_W32_Shared
   Arena *arena;
   String8List env_strings;
   
+  // rjf: run/mem/reg gens
+  U64 run_gen;
+  U64 mem_gen;
+  U64 reg_gen;
+  
   // rjf: detaching info
   Arena *detach_arena;
   DMN_HandleList detach_processes;

src/df/core/df_core.c

@@ -2895,7 +2895,7 @@ df_trap_net_from_thread__step_over_line(Arena *arena, DF_Entity *thread)
   String8 machine_code = {0};
   if(good_line_info)
   {
-    CTRL_ProcessMemorySlice machine_code_slice = ctrl_query_cached_data_from_process_vaddr_range(scratch.arena, process->ctrl_machine_id, process->ctrl_handle, line_vaddr_rng, os_now_microseconds()+5000);
+    CTRL_ProcessMemorySlice machine_code_slice = ctrl_query_cached_data_from_process_vaddr_range(scratch.arena, process->ctrl_machine_id, process->ctrl_handle, line_vaddr_rng, os_now_microseconds()+50000);
     machine_code = machine_code_slice.data;
   }
   
@@ -3656,7 +3656,7 @@ internal CTRL_Unwind
 df_push_unwind_from_thread(Arena *arena, DF_Entity *thread)
 {
   DF_Entity *process = df_entity_ancestor_from_kind(thread, DF_EntityKind_Process);
-  CTRL_Unwind unwind = ctrl_unwind_from_thread(arena, df_state->ctrl_entity_store, thread->ctrl_machine_id, thread->ctrl_handle, 0);
+  CTRL_Unwind unwind = ctrl_unwind_from_thread(arena, df_state->ctrl_entity_store, thread->ctrl_machine_id, thread->ctrl_handle, os_now_microseconds()+5000);
   return unwind;
 }
 
@@ -6545,8 +6545,8 @@ df_core_begin_frame(Arena *arena, DF_CmdList *cmds, F32 dt)
     Temp scratch = scratch_begin(&arena, 1);
     
     //- rjf: grab next reggen/memgen
-    U64 new_memgen_idx = ctrl_memgen_idx();
+    U64 new_mem_gen = ctrl_mem_gen();
-    U64 new_reggen_idx = ctrl_reggen_idx();
+    U64 new_reg_gen = ctrl_reg_gen();
     
     //- rjf: consume & process events
     CTRL_EventList events = ctrl_c2u_pop_events(scratch.arena);
@@ -6908,8 +6908,8 @@ df_core_begin_frame(Arena *arena, DF_CmdList *cmds, F32 dt)
     }
     
     //- rjf: refresh unwind cache
-    if((df_state->unwind_cache_memgen_idx != new_memgen_idx ||
+    if((df_state->unwind_cache_memgen_idx != new_mem_gen ||
-        df_state->unwind_cache_reggen_idx != new_reggen_idx) &&
+        df_state->unwind_cache_reggen_idx != new_reg_gen) &&
        !df_ctrl_targets_running()) ProfScope("per-thread unwind gather")
     {
       B32 good = 1;
@@ -6935,41 +6935,44 @@ df_core_begin_frame(Arena *arena, DF_CmdList *cmds, F32 dt)
           break;
         }
       }
-      df_state->unwind_cache_memgen_idx = new_memgen_idx;
+      if(good)
-      df_state->unwind_cache_reggen_idx = new_reggen_idx;
+      {
+        df_state->unwind_cache_memgen_idx = new_mem_gen;
+        df_state->unwind_cache_reggen_idx = new_reg_gen;
+      }
     }
     
     //- rjf: clear tls base cache
-    if((df_state->tls_base_cache_reggen_idx != new_reggen_idx ||
+    if((df_state->tls_base_cache_reggen_idx != new_reg_gen ||
-        df_state->tls_base_cache_memgen_idx != new_memgen_idx) &&
+        df_state->tls_base_cache_memgen_idx != new_mem_gen) &&
        !df_ctrl_targets_running())
     {
       DF_RunTLSBaseCache *cache = &df_state->tls_base_cache;
       arena_clear(cache->arena);
       cache->slots_count = 0;
       cache->slots = 0;
-      df_state->tls_base_cache_reggen_idx = new_reggen_idx;
+      df_state->tls_base_cache_reggen_idx = new_reg_gen;
-      df_state->tls_base_cache_memgen_idx = new_memgen_idx;
+      df_state->tls_base_cache_memgen_idx = new_mem_gen;
     }
     
     //- rjf: clear locals cache
-    if(df_state->locals_cache_reggen_idx != new_reggen_idx && !df_ctrl_targets_running())
+    if(df_state->locals_cache_reggen_idx != new_reg_gen && !df_ctrl_targets_running())
     {
       DF_RunLocalsCache *cache = &df_state->locals_cache;
       arena_clear(cache->arena);
       cache->table_size = 0;
       cache->table = 0;
-      df_state->locals_cache_reggen_idx = new_reggen_idx;
+      df_state->locals_cache_reggen_idx = new_reg_gen;
     }
     
     //- rjf: clear members cache
-    if(df_state->member_cache_reggen_idx != new_reggen_idx && !df_ctrl_targets_running())
+    if(df_state->member_cache_reggen_idx != new_reg_gen && !df_ctrl_targets_running())
     {
       DF_RunLocalsCache *cache = &df_state->member_cache;
       arena_clear(cache->arena);
       cache->table_size = 0;
       cache->table = 0;
-      df_state->member_cache_reggen_idx = new_reggen_idx;
+      df_state->member_cache_reggen_idx = new_reg_gen;
     }
     
     scratch_end(scratch);

src/df/gfx/df_gfx.c

@@ -3111,8 +3111,8 @@ df_window_update_and_render(Arena *arena, OS_EventList *events, DF_Window *ws, D
             avg_ui_hash_chain_length = chain_length_sum / chain_count;
           }
           ui_labelf("Target Hz: %.2f", 1.f/df_dt());
-          ui_labelf("Ctrl Run Index: %I64u", ctrl_run_idx());
+          ui_labelf("Ctrl Run Index: %I64u", ctrl_run_gen());
-          ui_labelf("Ctrl Mem Gen Index: %I64u", ctrl_memgen_idx());
+          ui_labelf("Ctrl Mem Gen Index: %I64u", ctrl_mem_gen());
           ui_labelf("Window %p", window);
           ui_set_next_pref_width(ui_children_sum(1));
           ui_set_next_pref_height(ui_children_sum(1));

src/df/gfx/df_view_rule_hooks.c

@@ -565,7 +565,7 @@ DF_GFX_VIEW_RULE_BLOCK_UI_FUNCTION_DEF(rgba)
   Vec4F32 rgba = {0};
   Vec4F32 hsva = {0};
   {
-    if(state->memgen_idx >= ctrl_memgen_idx())
+    if(state->memgen_idx >= ctrl_mem_gen())
     {
       hsva = state->hsva;
       rgba = rgba_from_hsva(hsva);
@@ -575,7 +575,7 @@ DF_GFX_VIEW_RULE_BLOCK_UI_FUNCTION_DEF(rgba)
       DF_Eval value_eval = df_value_mode_eval_from_eval(parse_ctx->type_graph, parse_ctx->rdi, ctrl_ctx, eval);
       rgba = df_view_rule_hooks__rgba_from_eval(value_eval, parse_ctx->type_graph, parse_ctx->rdi, process);
       state->hsva = hsva = hsva_from_rgba(rgba);
-      state->memgen_idx = ctrl_memgen_idx();
+      state->memgen_idx = ctrl_mem_gen();
     }
   }
   Vec4F32 initial_hsva = hsva;
@@ -624,7 +624,7 @@ DF_GFX_VIEW_RULE_BLOCK_UI_FUNCTION_DEF(rgba)
   {
     Vec4F32 rgba = rgba_from_hsva(hsva);
     df_view_rule_hooks__eval_commit_rgba(eval, parse_ctx->type_graph, parse_ctx->rdi, ctrl_ctx, rgba);
-    state->memgen_idx = ctrl_memgen_idx();
+    state->memgen_idx = ctrl_mem_gen();
   }
   
   //- rjf: commit possible edited value to state

src/df/gfx/df_views.c

@@ -7756,7 +7756,7 @@ DF_VIEW_UI_FUNCTION_DEF(Memory)
   U8 *visible_memory = 0;
   {
     Rng1U64 chunk_aligned_range_bytes = r1u64(AlignDownPow2(viz_range_bytes.min, KB(4)), AlignPow2(viz_range_bytes.max, KB(4)));
-    U64 current_memgen_idx = ctrl_memgen_idx();
+    U64 current_memgen_idx = ctrl_mem_gen();
     B32 range_changed = (chunk_aligned_range_bytes.min != mv->last_viewed_memory_cache_range.min ||
                          chunk_aligned_range_bytes.max != mv->last_viewed_memory_cache_range.max);
     B32 mem_changed = (current_memgen_idx != mv->last_viewed_memory_cache_memgen_idx);