raddebugger/src/geo_cache/geo_cache.c

// Copyright (c) 2024 Epic Games Tools
// Licensed under the MIT license (https://opensource.org/license/mit/)

////////////////////////////////
//~ rjf: Main Layer Initialization

internal void
geo_init(void)
{
  Arena *arena = arena_alloc();
  geo_shared = push_array(arena, GEO_Shared, 1);
  geo_shared->arena = arena;
  geo_shared->slots_count = 1024;
  geo_shared->stripes_count = Min(geo_shared->slots_count, os_logical_core_count());
  geo_shared->slots = push_array(arena, GEO_Slot, geo_shared->slots_count);
  geo_shared->stripes = push_array(arena, GEO_Stripe, geo_shared->stripes_count);
  geo_shared->stripes_free_nodes = push_array(arena, GEO_Node *, geo_shared->stripes_count);
  for(U64 idx = 0; idx < geo_shared->stripes_count; idx += 1)
  {
    geo_shared->stripes[idx].arena = arena_alloc();
    geo_shared->stripes[idx].rw_mutex = os_rw_mutex_alloc();
    geo_shared->stripes[idx].cv = os_condition_variable_alloc();
  }
  geo_shared->u2x_ring_size = KB(64);
  geo_shared->u2x_ring_base = push_array_no_zero(arena, U8, geo_shared->u2x_ring_size);
  geo_shared->u2x_ring_cv = os_condition_variable_alloc();
  geo_shared->u2x_ring_mutex = os_mutex_alloc();
  geo_shared->xfer_thread_count = Clamp(1, os_logical_core_count()-1, 4);
  geo_shared->xfer_threads = push_array(arena, OS_Handle, geo_shared->xfer_thread_count);
  for(U64 idx = 0; idx < geo_shared->xfer_thread_count; idx += 1)
  {
    geo_shared->xfer_threads[idx] = os_launch_thread(geo_xfer_thread__entry_point, (void *)idx, 0);
  }
  geo_shared->evictor_thread = os_launch_thread(geo_evictor_thread__entry_point, 0, 0);
}

////////////////////////////////
//~ rjf: Thread Context Initialization

internal void
geo_tctx_ensure_inited(void)
{
  if(geo_tctx == 0)
  {
    Arena *arena = arena_alloc();
    geo_tctx = push_array(arena, GEO_TCTX, 1);
    geo_tctx->arena = arena;
  }
}

////////////////////////////////
//~ rjf: User Clock

internal void
geo_user_clock_tick(void)
{
  ins_atomic_u64_inc_eval(&geo_shared->user_clock_idx);
}

internal U64
geo_user_clock_idx(void)
{
  return ins_atomic_u64_eval(&geo_shared->user_clock_idx);
}

////////////////////////////////
//~ rjf: Scoped Access

internal GEO_Scope *
geo_scope_open(void)
{
  geo_tctx_ensure_inited();
  GEO_Scope *scope = geo_tctx->free_scope;
  if(scope)
  {
    SLLStackPop(geo_tctx->free_scope);
  }
  else
  {
    scope = push_array_no_zero(geo_tctx->arena, GEO_Scope, 1);
  }
  MemoryZeroStruct(scope);
  return scope;
}

internal void
geo_scope_close(GEO_Scope *scope)
{
  for(GEO_Touch *touch = scope->top_touch, *next = 0; touch != 0; touch = next)
  {
    U128 hash = touch->hash;
    next = touch->next;
    U64 slot_idx = hash.u64[1]%geo_shared->slots_count;
    U64 stripe_idx = slot_idx%geo_shared->stripes_count;
    GEO_Slot *slot = &geo_shared->slots[slot_idx];
    GEO_Stripe *stripe = &geo_shared->stripes[stripe_idx];
    OS_MutexScopeR(stripe->rw_mutex)
    {
      for(GEO_Node *n = slot->first; n != 0; n = n->next)
      {
        if(u128_match(hash, n->hash))
        {
          ins_atomic_u64_dec_eval(&n->scope_ref_count);
          break;
        }
      }
    }
    SLLStackPush(geo_tctx->free_touch, touch);
  }
  SLLStackPush(geo_tctx->free_scope, scope);
}

internal void
geo_scope_touch_node__stripe_r_guarded(GEO_Scope *scope, GEO_Node *node)
{
  GEO_Touch *touch = geo_tctx->free_touch;
  ins_atomic_u64_inc_eval(&node->scope_ref_count);
  ins_atomic_u64_eval_assign(&node->last_time_touched_us, os_now_microseconds());
  ins_atomic_u64_eval_assign(&node->last_user_clock_idx_touched, geo_user_clock_idx());
  if(touch != 0)
  {
    SLLStackPop(geo_tctx->free_touch);
  }
  else
  {
    touch = push_array_no_zero(geo_tctx->arena, GEO_Touch, 1);
  }
  MemoryZeroStruct(touch);
  touch->hash = node->hash;
  SLLStackPush(scope->top_touch, touch);
}

////////////////////////////////
//~ rjf: Cache Lookups

internal R_Handle
geo_buffer_from_hash(GEO_Scope *scope, U128 hash)
{
  R_Handle handle = {0};
  if(!u128_match(hash, u128_zero()))
  {
    U64 slot_idx = hash.u64[1]%geo_shared->slots_count;
    U64 stripe_idx = slot_idx%geo_shared->stripes_count;
    GEO_Slot *slot = &geo_shared->slots[slot_idx];
    GEO_Stripe *stripe = &geo_shared->stripes[stripe_idx];
    B32 found = 0;
    OS_MutexScopeR(stripe->rw_mutex)
    {
      for(GEO_Node *n = slot->first; n != 0; n = n->next)
      {
        if(u128_match(hash, n->hash))
        {
          handle = n->buffer;
          found = !r_handle_match(r_handle_zero(), handle);
          geo_scope_touch_node__stripe_r_guarded(scope, n);
          break;
        }
      }
    }
    B32 node_is_new = 0;
    if(!found)
    {
      OS_MutexScopeW(stripe->rw_mutex)
      {
        GEO_Node *node = 0;
        for(GEO_Node *n = slot->first; n != 0; n = n->next)
        {
          if(u128_match(hash, n->hash))
          {
            node = n;
            break;
          }
        }
        if(node == 0)
        {
          node = geo_shared->stripes_free_nodes[stripe_idx];
          if(node)
          {
            SLLStackPop(geo_shared->stripes_free_nodes[stripe_idx]);
          }
          else
          {
            node = push_array_no_zero(stripe->arena, GEO_Node, 1);
          }
          MemoryZeroStruct(node);
          DLLPushBack(slot->first, slot->last, node);
          node->hash = hash;
          node_is_new = 1;
        }
      }
    }
    if(node_is_new)
    {
      geo_u2x_enqueue_req(hash, max_U64);
    }
  }
  return handle;
}

internal R_Handle
geo_buffer_from_key(GEO_Scope *scope, U128 key)
{
  R_Handle handle = {0};
  for(U64 rewind_idx = 0; rewind_idx < 2; rewind_idx += 1)
  {
    U128 hash = hs_hash_from_key(key, rewind_idx);
    handle = geo_buffer_from_hash(scope, hash);
    if(!r_handle_match(handle, r_handle_zero()))
    {
      break;
    }
  }
  return handle;
}

////////////////////////////////
//~ rjf: Transfer Threads

internal B32
geo_u2x_enqueue_req(U128 hash, U64 endt_us)
{
  B32 good = 0;
  OS_MutexScope(geo_shared->u2x_ring_mutex) for(;;)
  {
    U64 unconsumed_size = geo_shared->u2x_ring_write_pos-geo_shared->u2x_ring_read_pos;
    U64 available_size = geo_shared->u2x_ring_size-unconsumed_size;
    if(available_size >= sizeof(hash))
    {
      good = 1;
      geo_shared->u2x_ring_write_pos += ring_write_struct(geo_shared->u2x_ring_base, geo_shared->u2x_ring_size, geo_shared->u2x_ring_write_pos, &hash);
      break;
    }
    if(os_now_microseconds() >= endt_us)
    {
      break;
    }
    os_condition_variable_wait(geo_shared->u2x_ring_cv, geo_shared->u2x_ring_mutex, endt_us);
  }
  if(good)
  {
    os_condition_variable_broadcast(geo_shared->u2x_ring_cv);
  }
  return good;
}

internal void
geo_u2x_dequeue_req(U128 *hash_out)
{
  OS_MutexScope(geo_shared->u2x_ring_mutex) for(;;)
  {
    U64 unconsumed_size = geo_shared->u2x_ring_write_pos-geo_shared->u2x_ring_read_pos;
    if(unconsumed_size >= sizeof(*hash_out))
    {
      geo_shared->u2x_ring_read_pos += ring_read_struct(geo_shared->u2x_ring_base, geo_shared->u2x_ring_size, geo_shared->u2x_ring_read_pos, hash_out);
      break;
    }
    os_condition_variable_wait(geo_shared->u2x_ring_cv, geo_shared->u2x_ring_mutex, max_U64);
  }
  os_condition_variable_broadcast(geo_shared->u2x_ring_cv);
}

internal void
geo_xfer_thread__entry_point(void *p)
{
  for(;;)
  {
    HS_Scope *scope = hs_scope_open();
    
    //- rjf: decode
    U128 hash = {0};
    geo_u2x_dequeue_req(&hash);
    
    //- rjf: unpack hash
    U64 slot_idx = hash.u64[1]%geo_shared->slots_count;
    U64 stripe_idx = slot_idx%geo_shared->stripes_count;
    GEO_Slot *slot = &geo_shared->slots[slot_idx];
    GEO_Stripe *stripe = &geo_shared->stripes[stripe_idx];
    
    //- rjf: take task
    B32 got_task = 0;
    OS_MutexScopeR(stripe->rw_mutex)
    {
      for(GEO_Node *n = slot->first; n != 0; n = n->next)
      {
        if(u128_match(n->hash, hash))
        {
          got_task = !ins_atomic_u32_eval_cond_assign(&n->is_working, 1, 0);
          break;
        }
      }
    }
    
    //- rjf: hash -> data
    String8 data = {0};
    if(got_task)
    {
      data = hs_data_from_hash(scope, hash);
    }
    
    //- rjf: data -> buffer
    R_Handle buffer = {0};
    if(got_task && data.size != 0)
    {
      buffer = r_buffer_alloc(R_BufferKind_Static, data.size, data.str);
    }
    
    //- rjf: commit results to cache
    if(got_task) OS_MutexScopeW(stripe->rw_mutex)
    {
      for(GEO_Node *n = slot->first; n != 0; n = n->next)
      {
        if(u128_match(n->hash, hash))
        {
          n->buffer = buffer;
          ins_atomic_u32_eval_assign(&n->is_working, 0);
          ins_atomic_u64_inc_eval(&n->load_count);
          break;
        }
      }
    }
    
    hs_scope_close(scope);
  }
}

////////////////////////////////
//~ rjf: Evictor Threads

internal void
geo_evictor_thread__entry_point(void *p)
{
  for(;;)
  {
    U64 check_time_us = os_now_microseconds();
    U64 check_time_user_clocks = geo_user_clock_idx();
    U64 evict_threshold_us = 10*1000000;
    U64 evict_threshold_user_clocks = 10;
    for(U64 slot_idx = 0; slot_idx < geo_shared->slots_count; slot_idx += 1)
    {
      U64 stripe_idx = slot_idx%geo_shared->stripes_count;
      GEO_Slot *slot = &geo_shared->slots[slot_idx];
      GEO_Stripe *stripe = &geo_shared->stripes[stripe_idx];
      B32 slot_has_work = 0;
      OS_MutexScopeR(stripe->rw_mutex)
      {
        for(GEO_Node *n = slot->first; n != 0; n = n->next)
        {
          if(n->scope_ref_count == 0 &&
             n->last_time_touched_us+evict_threshold_us <= check_time_us &&
             n->last_user_clock_idx_touched+evict_threshold_user_clocks <= check_time_user_clocks &&
             n->load_count != 0 &&
             n->is_working == 0)
          {
            slot_has_work = 1;
            break;
          }
        }
      }
      if(slot_has_work) OS_MutexScopeW(stripe->rw_mutex)
      {
        for(GEO_Node *n = slot->first, *next = 0; n != 0; n = next)
        {
          next = n->next;
          if(n->scope_ref_count == 0 &&
             n->last_time_touched_us+evict_threshold_us <= check_time_us &&
             n->last_user_clock_idx_touched+evict_threshold_user_clocks <= check_time_user_clocks &&
             n->load_count != 0 &&
             n->is_working == 0)
          {
            DLLRemove(slot->first, slot->last, n);
            if(!r_handle_match(n->buffer, r_handle_zero()))
            {
              r_buffer_release(n->buffer);
            }
            SLLStackPush(geo_shared->stripes_free_nodes[stripe_idx], n);
          }
        }
      }
      os_sleep_milliseconds(5);
    }
    os_sleep_milliseconds(1000);
  }
}