progress on single async-filled computation artifact cache; hook up to text cache layer, eliminate bespoke cache for text only

src/artifact_cache/artifact_cache.c

@@ -13,6 +13,8 @@ ac_init(void)
   ac_shared->cache_slots_count = 256;
   ac_shared->cache_slots = push_array(arena, AC_Cache *, ac_shared->cache_slots_count);
   ac_shared->cache_stripes = stripe_array_alloc(arena);
+  ac_shared->req_mutex = mutex_alloc();
+  ac_shared->req_arena = arena_alloc();
 }
 
 ////////////////////////////////
@@ -88,12 +90,13 @@ ac_artifact_from_key(Access *access, String8 key, AC_CreateFunctionType *create,
           }
           else
           {
-            node = push_array(stripe->arena, AC_Node, 1);
+            node = push_array_no_zero(stripe->arena, AC_Node, 1);
-            DLLPushBack(slot->first, slot->last, node);
-            // TODO(rjf): string allocator for keys
-            node->key = str8_copy(stripe->arena, key);
-            node->working_count = 1;
           }
+          MemoryZeroStruct(node);
+          DLLPushBack(slot->first, slot->last, node);
+          // TODO(rjf): string allocator for keys
+          node->key = str8_copy(stripe->arena, key);
+          node->working_count = 1;
         }
       }
       if(found)
@@ -110,6 +113,7 @@ ac_artifact_from_key(Access *access, String8 key, AC_CreateFunctionType *create,
           n->v.key = str8_copy(ac_shared->req_arena, key);
           n->v.create = create;
         }
+        cond_var_broadcast(async_tick_start_cond_var);
       }
     }
   }
@@ -199,7 +203,49 @@ ac_tick(void)
   //- rjf: do all requests on all lanes
   for EachIndex(idx, reqs_count)
   {
-    reqs[idx].create(reqs[idx].key);
+    lane_sync();
+    
+    // rjf: compute val
+    void *val = reqs[idx].create(reqs[idx].key);
+    
+    // rjf: create function -> cache
+    AC_Cache *cache = 0;
+    {
+      U64 cache_hash = u64_hash_from_str8(str8_struct(&reqs[idx].create));
+      U64 cache_slot_idx = cache_hash%ac_shared->cache_slots_count;
+      Stripe *cache_stripe = stripe_from_slot_idx(&ac_shared->cache_stripes, cache_slot_idx);
+      RWMutexScope(cache_stripe->rw_mutex, 0)
+      {
+        for(AC_Cache *c = ac_shared->cache_slots[cache_slot_idx]; c != 0; c = c->next)
+        {
+          if(c->create == reqs[idx].create)
+          {
+            cache = c;
+            break;
+          }
+        }
+      }
+    }
+    
+    // rjf: write value into cache
+    if(lane_idx() == 0)
+    {
+      U64 hash = u64_hash_from_str8(reqs[idx].key);
+      U64 slot_idx = hash%cache->slots_count;
+      AC_Slot *slot = &cache->slots[slot_idx];
+      Stripe *stripe = stripe_from_slot_idx(&cache->stripes, slot_idx);
+      RWMutexScope(stripe->rw_mutex, 1)
+      {
+        for(AC_Node *n = slot->first; n != 0; n = n->next)
+        {
+          if(str8_match(n->key, reqs[idx].key, 0))
+          {
+            n->val = val;
+            ins_atomic_u64_dec_eval(&n->working_count);
+          }
+        }
+      }
+    }
   }
   lane_sync();
   

src/base/base_entry_point.c

@@ -65,9 +65,6 @@ main_thread_base_entry_point(int arguments_count, char **arguments)
 #if defined(FILE_STREAM_H) && !defined(FS_INIT_MANUAL)
   fs_init();
 #endif
-#if defined(TEXT_CACHE_H) && !defined(TXT_INIT_MANUAL)
-  txt_init();
-#endif
 #if defined(MUTABLE_TEXT_H) && !defined(MTX_INIT_MANUAL)
   mtx_init();
 #endif

src/base/base_thread_context.h

@@ -65,6 +65,10 @@ struct TCTX
   Arena *access_arena;
   Access *free_access;
   Touch *free_touch;
+  
+  // rjf: progress
+  U64 *progress_counter_ptr;
+  U64 *progress_target_ptr;
 };
 
 ////////////////////////////////
@@ -108,4 +112,11 @@ internal void access_touch(Access *access, AccessPt *pt, CondVar cv);
 //- rjf: access points
 internal B32 access_pt_is_expired(AccessPt *pt);
 
+//- rjf: progress counters
+#define set_progress_ptr(ptr) (tctx_selected()->progress_counter_ptr = (ptr))
+#define set_progress_target_ptr(ptr) (tctx_selected()->progress_target_ptr = (ptr))
+#define set_progress(val) (tctx_selected()->progress_counter_ptr ? ins_atomic_u64_eval_assign(tctx_selected()->progress_counter_ptr, (val)) : (void)0)
+#define add_progress(val) (tctx_selected()->progress_counter_ptr ? ins_atomic_u64_add_eval(tctx_selected()->progress_counter_ptr, (val)) : (void)0)
+#define set_progress_target(val) (tctx_selected()->progress_target_ptr ? ins_atomic_u64_eval_assign(tctx_selected()->progress_target_ptr, (val)) : (void)0)
+
 #endif // BASE_THREAD_CONTEXT_H

src/text_cache/text_cache.c

@@ -1590,117 +1590,6 @@ txt_token_array_from_string__disasm_x64_intel(Arena *arena, U64 *bytes_processed
   return result;
 }
 
-////////////////////////////////
-//~ rjf: Main Layer Initialization
-
-internal void
-txt_init(void)
-{
-  Arena *arena = arena_alloc();
-  txt_shared = push_array(arena, TXT_Shared, 1);
-  txt_shared->arena = arena;
-  txt_shared->slots_count = 1024;
-  txt_shared->slots = push_array(arena, TXT_Slot, txt_shared->slots_count);
-  txt_shared->stripes = stripe_array_alloc(arena);
-  txt_shared->stripes_free_nodes = push_array(arena, TXT_Node *, txt_shared->stripes.count);
-  txt_shared->u2p_ring_size = KB(64);
-  txt_shared->u2p_ring_base = push_array_no_zero(arena, U8, txt_shared->u2p_ring_size);
-  txt_shared->u2p_ring_cv = cond_var_alloc();
-  txt_shared->u2p_ring_mutex = mutex_alloc();
-  txt_shared->evictor_thread = thread_launch(txt_evictor_thread__entry_point, 0);
-}
-
-////////////////////////////////
-//~ rjf: Cache Lookups
-
-internal TXT_TextInfo
-txt_text_info_from_hash_lang(Access *access, U128 hash, TXT_LangKind lang)
-{
-  TXT_TextInfo info = {0};
-  if(!u128_match(hash, u128_zero()))
-  {
-    U64 slot_idx = hash.u64[1]%txt_shared->slots_count;
-    TXT_Slot *slot = &txt_shared->slots[slot_idx];
-    Stripe *stripe = stripe_from_slot_idx(&txt_shared->stripes, slot_idx);
-    U64 stripe_idx = (stripe - txt_shared->stripes.v);
-    B32 found = 0;
-    MutexScopeR(stripe->rw_mutex)
-    {
-      for(TXT_Node *n = slot->first; n != 0; n = n->next)
-      {
-        if(u128_match(hash, n->hash) && n->lang == lang)
-        {
-          MemoryCopyStruct(&info, &n->info);
-          info.bytes_processed = ins_atomic_u64_eval(&n->info.bytes_processed);
-          info.bytes_to_process = ins_atomic_u64_eval(&n->info.bytes_to_process);
-          found = 1;
-          access_touch(access, &n->access_pt, stripe->cv);
-          break;
-        }
-      }
-    }
-    B32 node_is_new = 0;
-    if(!found)
-    {
-      MutexScopeW(stripe->rw_mutex)
-      {
-        TXT_Node *node = 0;
-        for(TXT_Node *n = slot->first; n != 0; n = n->next)
-        {
-          if(u128_match(hash, n->hash) && n->lang == lang)
-          {
-            node = n;
-            break;
-          }
-        }
-        if(node == 0)
-        {
-          node = txt_shared->stripes_free_nodes[stripe_idx];
-          if(node)
-          {
-            SLLStackPop(txt_shared->stripes_free_nodes[stripe_idx]);
-          }
-          else
-          {
-            node = push_array_no_zero(stripe->arena, TXT_Node, 1);
-          }
-          MemoryZeroStruct(node);
-          DLLPushBack(slot->first, slot->last, node);
-          node->hash = hash;
-          node->lang = lang;
-          node_is_new = 1;
-        }
-      }
-    }
-    if(node_is_new)
-    {
-      txt_u2p_enqueue_req(hash, lang, max_U64);
-      async_push_work(txt_parse_work);
-    }
-  }
-  return info;
-}
-
-internal TXT_TextInfo
-txt_text_info_from_key_lang(Access *access, C_Key key, TXT_LangKind lang, U128 *hash_out)
-{
-  TXT_TextInfo result = {0};
-  for(U64 rewind_idx = 0; rewind_idx < C_KEY_HASH_HISTORY_COUNT; rewind_idx += 1)
-  {
-    U128 hash = c_hash_from_key(key, rewind_idx);
-    result = txt_text_info_from_hash_lang(access, hash, lang);
-    if(result.lines_count != 0)
-    {
-      if(hash_out)
-      {
-        *hash_out = hash;
-      }
-      break;
-    }
-  }
-  return result;
-}
-
 ////////////////////////////////
 //~ rjf: Text Info Extractor Helpers
 
@@ -2062,120 +1951,61 @@ txt_scope_node_from_info_pt(TXT_TextInfo *info, TxtPt pt)
 }
 
 ////////////////////////////////
-//~ rjf: Transfer Threads
+//~ rjf: Artifact Cache Hooks / Lookups
 
-internal B32
+typedef struct TXT_Artifact TXT_Artifact;
-txt_u2p_enqueue_req(U128 hash, TXT_LangKind lang, U64 endt_us)
+struct TXT_Artifact
 {
-  B32 good = 0;
+  Arena *arena;
-  MutexScope(txt_shared->u2p_ring_mutex) for(;;)
+  TXT_TextInfo info;
-  {
+};
-    U64 unconsumed_size = txt_shared->u2p_ring_write_pos - txt_shared->u2p_ring_read_pos;
-    U64 available_size = txt_shared->u2p_ring_size - unconsumed_size;
-    if(available_size >= sizeof(hash)+sizeof(lang))
-    {
-      good = 1;
-      txt_shared->u2p_ring_write_pos += ring_write_struct(txt_shared->u2p_ring_base, txt_shared->u2p_ring_size, txt_shared->u2p_ring_write_pos, &hash);
-      txt_shared->u2p_ring_write_pos += ring_write_struct(txt_shared->u2p_ring_base, txt_shared->u2p_ring_size, txt_shared->u2p_ring_write_pos, &lang);
-      break;
-    }
-    if(os_now_microseconds() >= endt_us)
-    {
-      break;
-    }
-    cond_var_wait(txt_shared->u2p_ring_cv, txt_shared->u2p_ring_mutex, endt_us);
-  }
-  if(good)
-  {
-    cond_var_broadcast(txt_shared->u2p_ring_cv);
-  }
-  return good;
-}
 
-internal void
+typedef struct TXT_ArtifactCreateShared TXT_ArtifactCreateShared;
-txt_u2p_dequeue_req(U128 *hash_out, TXT_LangKind *lang_out)
+struct TXT_ArtifactCreateShared
 {
-  MutexScope(txt_shared->u2p_ring_mutex) for(;;)
+  Arena *arena;
-  {
+  TXT_TextInfo info;
-    U64 unconsumed_size = txt_shared->u2p_ring_write_pos - txt_shared->u2p_ring_read_pos;
+  TXT_Artifact *artifact;
-    if(unconsumed_size >= sizeof(*hash_out) + sizeof(*lang_out))
+};
-    {
-      txt_shared->u2p_ring_read_pos += ring_read_struct(txt_shared->u2p_ring_base, txt_shared->u2p_ring_size, txt_shared->u2p_ring_read_pos, hash_out);
-      txt_shared->u2p_ring_read_pos += ring_read_struct(txt_shared->u2p_ring_base, txt_shared->u2p_ring_size, txt_shared->u2p_ring_read_pos, lang_out);
-      break;
-    }
-    cond_var_wait(txt_shared->u2p_ring_cv, txt_shared->u2p_ring_mutex, max_U64);
-  }
-  cond_var_broadcast(txt_shared->u2p_ring_cv);
-}
 
-ASYNC_WORK_DEF(txt_parse_work)
+internal void *
+txt_artifact_create(String8 key)
 {
   ProfBeginFunction();
-  
+  Temp scratch = scratch_begin(0, 0);
-  //- rjf: get next key
-  U128 hash = {0};
-  TXT_LangKind lang = TXT_LangKind_Null;
-  txt_u2p_dequeue_req(&hash, &lang);
   Access *access = access_open();
   
-  //- rjf: unpack hash
+  //- rjf: get shared state
-  U64 slot_idx = hash.u64[1]%txt_shared->slots_count;
+  local_persist TXT_ArtifactCreateShared *shared = 0;
-  TXT_Slot *slot = &txt_shared->slots[slot_idx];
+  if(lane_idx() == 0)
-  Stripe *stripe = stripe_from_slot_idx(&txt_shared->stripes, slot_idx);
-  
-  //- rjf: take task
-  B32 got_task = 0;
-  MutexScopeR(stripe->rw_mutex)
   {
-    for(TXT_Node *n = slot->first; n != 0; n = n->next)
+    shared = push_array(scratch.arena, TXT_ArtifactCreateShared, 1);
-    {
-      if(u128_match(n->hash, hash) && n->lang == lang)
-      {
-        got_task = !ins_atomic_u32_eval_cond_assign(&n->is_working, 1, 0);
-        break;
-      }
-    }
   }
+  lane_sync();
   
-  //- rjf: hash -> data
+  //- rjf: unpack key
-  String8 data = {0};
+  U128 hash = {0};
-  if(got_task)
+  TXT_LangKind lang = TXT_LangKind_Null;
-  {
+  str8_deserial_read_struct(key, 0, &hash);
-    data = c_data_from_hash(access, hash);
+  str8_deserial_read_struct(key, sizeof(hash), &lang);
-  }
+  String8 data = c_data_from_hash(access, hash);
+  TXT_LangLexFunctionType *lex_function = txt_lex_function_from_lang_kind(lang);
   
   //- rjf: data -> text info
-  Arena *info_arena = 0;
+  if(!u128_match(hash, u128_zero()))
-  TXT_TextInfo info = {0};
-  if(got_task && !u128_match(hash, u128_zero()))
   {
-    info_arena = arena_alloc();
+    if(lane_idx() == 0)
-    
-    //- rjf: grab pointers to working counters
-    U64 *bytes_processed_ptr = 0;
-    U64 *bytes_to_process_ptr = 0;
-    MutexScopeR(stripe->rw_mutex)
     {
-      for(TXT_Node *n = slot->first; n != 0; n = n->next)
+      shared->arena = arena_alloc();
-      {
-        if(u128_match(n->hash, hash) && n->lang == lang)
-        {
-          bytes_processed_ptr = &n->info.bytes_processed;
-          bytes_to_process_ptr = &n->info.bytes_to_process;
-        }
-      }
     }
     
     //- rjf: set # of bytes to process
-    if(bytes_to_process_ptr)
+    //                  (line ending calc)     (line counting)    (line measuring)   (lexing)
-    {
+    set_progress_target(Min(data.size, 1024) + data.size        + data.size        + data.size*(lang != TXT_LangKind_Null));
-      //                                               (line ending calc)     (line counting)    (line measuring)   (lexing)
-      ins_atomic_u64_eval_assign(bytes_to_process_ptr, Min(data.size, 1024) + data.size        + data.size        + data.size*(lang != TXT_LangKind_Null));
-    }
     
     //- rjf: detect line end kind
     TXT_LineEndKind line_end_kind = TXT_LineEndKind_Null;
+    if(lane_idx() == 0)
     {
       U64 lf_count = 0;
       U64 cr_count = 0;
@@ -2198,323 +2028,255 @@ ASYNC_WORK_DEF(txt_parse_work)
       {
         line_end_kind = TXT_LineEndKind_LF;
       }
-      info.line_end_kind = line_end_kind;
+      shared->info.line_end_kind = line_end_kind;
-    }
-    
-    //- rjf: bump progress
-    if(bytes_processed_ptr)
-    {
-      ins_atomic_u64_eval_assign(bytes_processed_ptr, Min(data.size, 1024));
     }
+    lane_sync();
+    set_progress(Min(data.size, 1024));
     
     //- rjf: count # of lines
-    U64 line_count = 1;
+    U64 lane_line_count = 0;
-    U64 byte_process_start_idx = 0;
+    if(lane_idx() == 0)
-    for(U64 idx = 0; idx < data.size; idx += 1)
     {
-      if(data.str[idx] == '\n' || data.str[idx] == '\r')
+      lane_line_count = 1;
+    }
+    {
+      Rng1U64 range = lane_range(data.size);
+      for EachInRange(idx, range)
       {
-        line_count += 1;
+        if(data.str[idx] == '\n' || data.str[idx] == '\r')
-        if(data.str[idx] == '\r')
         {
-          idx += 1;
+          lane_line_count += 1;
+          if(data.str[idx] == '\r')
+          {
+            idx += 1;
+          }
+        }
+        if(idx && idx%1000 == 0)
+        {
+          add_progress(1000);
         }
       }
-      if(idx && idx%1000 == 0)
-      {
-        ins_atomic_u64_add_eval(bytes_processed_ptr, 1000);
-      }
-    }
-    
-    //- rjf: bump progress
-    if(bytes_processed_ptr)
-    {
-      ins_atomic_u64_eval_assign(bytes_processed_ptr, Min(data.size, 1024) + data.size);
     }
+    ins_atomic_u64_add_eval(&shared->info.lines_count, lane_line_count);
+    lane_sync();
+    set_progress(Min(data.size, 1024) + data.size);
     
     //- rjf: allocate & store line ranges
-    info.lines_count = line_count;
+    if(lane_idx() == 0)
-    info.lines_ranges = push_array_no_zero(info_arena, Rng1U64, info.lines_count);
-    U64 line_idx = 0;
-    U64 line_start_idx = 0;
-    for(U64 idx = 0; idx <= data.size; idx += 1)
     {
-      if(idx == data.size || data.str[idx] == '\n' || data.str[idx] == '\r')
+      shared->info.lines_ranges = push_array_no_zero(shared->arena, Rng1U64, shared->info.lines_count);
+      U64 line_idx = 0;
+      U64 line_start_idx = 0;
+      for(U64 idx = 0; idx <= data.size; idx += 1)
       {
-        Rng1U64 line_range = r1u64(line_start_idx, idx);
+        if(idx == data.size || data.str[idx] == '\n' || data.str[idx] == '\r')
-        U64 line_size = dim_1u64(line_range);
-        info.lines_ranges[line_idx] = line_range;
-        info.lines_max_size = Max(info.lines_max_size, line_size);
-        line_idx += 1;
-        line_start_idx = idx+1;
-        if(idx < data.size && data.str[idx] == '\r')
         {
-          line_start_idx += 1;
+          Rng1U64 line_range = r1u64(line_start_idx, idx);
-          idx += 1;
+          U64 line_size = dim_1u64(line_range);
+          shared->info.lines_ranges[line_idx] = line_range;
+          shared->info.lines_max_size = Max(shared->info.lines_max_size, line_size);
+          line_idx += 1;
+          line_start_idx = idx+1;
+          if(idx < data.size && data.str[idx] == '\r')
+          {
+            line_start_idx += 1;
+            idx += 1;
+          }
+        }
+        if(idx && idx%1000 == 0)
+        {
+          add_progress(1000);
         }
       }
-      if(idx && idx%1000 == 0)
-      {
-        ins_atomic_u64_add_eval(bytes_processed_ptr, 1000);
-      }
-    }
-    
-    //- rjf: bump progress
-    if(bytes_processed_ptr)
-    {
-      ins_atomic_u64_eval_assign(bytes_processed_ptr, Min(data.size, 1024) + data.size + data.size);
     }
+    lane_sync();
+    set_progress(Min(data.size, 1024) + data.size + data.size);
     
     //- rjf: lex function * data -> tokens
-    TXT_TokenArray tokens = {0};
+    if(lane_idx() == 0 && lex_function != 0)
-    TXT_LangLexFunctionType *lex_function = txt_lex_function_from_lang_kind(lang);
-    if(lex_function != 0)
     {
-      tokens = lex_function(info_arena, bytes_processed_ptr, data);
+      shared->info.tokens = lex_function(shared->arena, 0, data);
-    }
-    info.tokens = tokens;
-    
-    //- rjf: bump progress
-    if(bytes_processed_ptr)
-    {
-      ins_atomic_u64_eval_assign(bytes_processed_ptr, Min(data.size, 1024) + data.size + data.size + data.size*(lex_function != 0));
     }
+    lane_sync();
+    set_progress(Min(data.size, 1024) + data.size + data.size + data.size*(lex_function != 0));
+    TXT_TokenArray tokens = shared->info.tokens;
     
     //- rjf: count scope points
-    U64 scope_pt_opener_count = 0;
-    U64 scope_pt_count = 0;
-    for EachIndex(idx, tokens.count)
     {
-      if(tokens.v[idx].kind == TXT_TokenKind_Symbol)
+      U64 lane_scope_pt_opener_count = 0;
+      U64 lane_scope_pt_count = 0;
+      Rng1U64 range = lane_range(tokens.count);
+      for EachInRange(idx, range)
       {
-        String8 token_string = str8_substr(data, tokens.v[idx].range);
+        if(tokens.v[idx].kind == TXT_TokenKind_Symbol)
-        B32 is_opener = (token_string.str[0] == '{' ||
-                         token_string.str[0] == '(' ||
-                         token_string.str[0] == '[');
-        B32 is_closer = (token_string.str[0] == '}' ||
-                         token_string.str[0] == ')' ||
-                         token_string.str[0] == ']');
-        if(token_string.size == 1 && (is_opener || is_closer))
         {
-          scope_pt_count += 1;
+          String8 token_string = str8_substr(data, tokens.v[idx].range);
-          scope_pt_opener_count += !!is_opener;
-        }
-      }
-    }
-    
-    //- rjf: allocate & fill scope data
-    info.scope_pts.count = scope_pt_count;
-    info.scope_pts.v = push_array_no_zero(info_arena, TXT_ScopePt, info.scope_pts.count);
-    info.scope_nodes.count = scope_pt_opener_count;
-    info.scope_nodes.v = push_array_no_zero(info_arena, TXT_ScopeNode, info.scope_nodes.count);
-    {
-      typedef struct ScopeTask ScopeTask;
-      struct ScopeTask
-      {
-        ScopeTask *next;
-        U64 scope_idx;
-      };
-      Temp scratch = scratch_begin(0, 0);
-      ScopeTask *top_scope_task = 0;
-      ScopeTask *free_scope_task = 0;
-      U64 pt_idx = 0;
-      U64 scope_idx = 0;
-      for EachIndex(token_idx, tokens.count)
-      {
-        if(tokens.v[token_idx].kind == TXT_TokenKind_Symbol)
-        {
-          String8 token_string = str8_substr(data, tokens.v[token_idx].range);
           B32 is_opener = (token_string.str[0] == '{' ||
                            token_string.str[0] == '(' ||
                            token_string.str[0] == '[');
           B32 is_closer = (token_string.str[0] == '}' ||
                            token_string.str[0] == ')' ||
                            token_string.str[0] == ']');
-          
+          if(token_string.size == 1 && (is_opener || is_closer))
-          // rjf: opener symbols -> push scope
-          if(is_opener)
           {
-            // rjf: insert into scope tree
+            lane_scope_pt_count += 1;
-            TXT_ScopeNode *new_scope = &info.scope_nodes.v[scope_idx];
+            lane_scope_pt_opener_count += !!is_opener;
-            new_scope->token_idx_range.min = token_idx;
+          }
-            if(top_scope_task)
+        }
-            {
+      }
-              U64 new_scope_num = scope_idx+1;
+      ins_atomic_u64_add_eval(&shared->info.scope_pts.count, lane_scope_pt_count);
-              TXT_ScopeNode *parent = &info.scope_nodes.v[top_scope_task->scope_idx];
+      ins_atomic_u64_add_eval(&shared->info.scope_nodes.count, lane_scope_pt_opener_count);
-              if(parent->first_num == 0)
+    }
-              {
+    lane_sync();
-                parent->first_num = new_scope_num;
+    
-              }
+    //- rjf: allocate & fill scope data
-              if(parent->last_num != 0)
+    if(lane_idx() == 0)
-              {
+    {
-                TXT_ScopeNode *prev_scope = &info.scope_nodes.v[parent->last_num-1];
+      shared->info.scope_pts.v = push_array_no_zero(shared->arena, TXT_ScopePt, shared->info.scope_pts.count);
-                prev_scope->next_num = new_scope_num;
+      shared->info.scope_nodes.v = push_array_no_zero(shared->arena, TXT_ScopeNode, shared->info.scope_nodes.count);
-              }
+      {
-              parent->last_num = new_scope_num;
+        typedef struct ScopeTask ScopeTask;
-              new_scope->parent_num = top_scope_task->scope_idx+1;
+        struct ScopeTask
-            }
+        {
+          ScopeTask *next;
+          U64 scope_idx;
+        };
+        Temp scratch = scratch_begin(0, 0);
+        ScopeTask *top_scope_task = 0;
+        ScopeTask *free_scope_task = 0;
+        U64 pt_idx = 0;
+        U64 scope_idx = 0;
+        for EachIndex(token_idx, tokens.count)
+        {
+          if(tokens.v[token_idx].kind == TXT_TokenKind_Symbol)
+          {
+            String8 token_string = str8_substr(data, tokens.v[token_idx].range);
+            B32 is_opener = (token_string.str[0] == '{' ||
+                             token_string.str[0] == '(' ||
+                             token_string.str[0] == '[');
+            B32 is_closer = (token_string.str[0] == '}' ||
+                             token_string.str[0] == ')' ||
+                             token_string.str[0] == ']');
             
-            // rjf: push onto scope stack
+            // rjf: opener symbols -> push scope
-            ScopeTask *scope_task = free_scope_task;
+            if(is_opener)
-            if(scope_task)
             {
-              SLLStackPop(free_scope_task);
+              // rjf: insert into scope tree
-            }
+              TXT_ScopeNode *new_scope = &shared->info.scope_nodes.v[scope_idx];
-            else
+              new_scope->token_idx_range.min = token_idx;
-            {
+              if(top_scope_task)
-              scope_task = push_array(scratch.arena, ScopeTask, 1);
-            }
-            scope_task->scope_idx = scope_idx;
-            scope_idx += 1;
-            SLLStackPush(top_scope_task, scope_task);
-          }
-          
-          // rjf: opener or closer -> fill endpoint
-          if(top_scope_task && (is_opener || is_closer))
-          {
-            info.scope_pts.v[pt_idx].token_idx = token_idx;
-            info.scope_pts.v[pt_idx].scope_idx = top_scope_task->scope_idx;
-            pt_idx += 1;
-          }
-          
-          // rjf: closer symbols -> pop
-          if(is_closer && top_scope_task != 0)
-          {
-            ScopeTask *popped = top_scope_task;
-            info.scope_nodes.v[popped->scope_idx].token_idx_range.max = token_idx;
-            SLLStackPop(top_scope_task);
-            SLLStackPush(free_scope_task, popped);
-          }
-        }
-      }
-      scratch_end(scratch);
-    }
-  }
-  
-  //- rjf: commit results to cache
-  if(got_task) MutexScopeW(stripe->rw_mutex)
-  {
-    for(TXT_Node *n = slot->first; n != 0; n = n->next)
-    {
-      if(u128_match(n->hash, hash) && n->lang == lang)
-      {
-        c_hash_downstream_inc(n->hash);
-        n->arena = info_arena;
-        info.bytes_processed = n->info.bytes_processed;
-        info.bytes_to_process = n->info.bytes_to_process;
-        MemoryCopyStruct(&n->info, &info);
-        ins_atomic_u32_eval_assign(&n->is_working, 0);
-        ins_atomic_u64_inc_eval(&n->load_count);
-        break;
-      }
-    }
-  }
-  
-  access_close(access);
-  ProfEnd();
-  return 0;
-}
-
-////////////////////////////////
-//~ rjf: Evictor Threads
-
-internal void
-txt_evictor_thread__entry_point(void *p)
-{
-  ThreadNameF("txt_evictor_thread");
-  for(;;)
-  {
-    for(U64 slot_idx = 0; slot_idx < txt_shared->slots_count; slot_idx += 1)
-    {
-      TXT_Slot *slot = &txt_shared->slots[slot_idx];
-      Stripe *stripe = stripe_from_slot_idx(&txt_shared->stripes, slot_idx);
-      U64 stripe_idx = (stripe - txt_shared->stripes.v);
-      B32 slot_has_work = 0;
-      MutexScopeR(stripe->rw_mutex)
-      {
-        for(TXT_Node *n = slot->first; n != 0; n = n->next)
-        {
-          if(access_pt_is_expired(&n->access_pt) &&
-             n->load_count != 0 &&
-             n->is_working == 0)
-          {
-            slot_has_work = 1;
-            break;
-          }
-        }
-      }
-      if(slot_has_work) MutexScopeW(stripe->rw_mutex)
-      {
-        for(TXT_Node *n = slot->first, *next = 0; n != 0; n = next)
-        {
-          next = n->next;
-          if(access_pt_is_expired(&n->access_pt) &&
-             n->load_count != 0 &&
-             n->is_working == 0)
-          {
-            DLLRemove(slot->first, slot->last, n);
-            c_hash_downstream_dec(n->hash);
-            if(n->arena != 0)
-            {
-              arena_release(n->arena);
-            }
-            SLLStackPush(txt_shared->stripes_free_nodes[stripe_idx], n);
-          }
-        }
-      }
-    }
-    os_sleep_milliseconds(500);
-  }
-}
-
-////////////////////////////////
-//~ rjf: Tick
-
-internal void
-txt_tick(void)
-{
-  //- rjf: do eviction pass
-  {
-    Rng1U64 range = lane_range(txt_shared->slots_count);
-    for EachInRange(slot_idx, range)
-    {
-      TXT_Slot *slot = &txt_shared->slots[slot_idx];
-      Stripe *stripe = stripe_from_slot_idx(&txt_shared->stripes, slot_idx);
-      for(B32 write_mode = 0; write_mode <= 1; write_mode += 1)
-      {
-        B32 slot_has_work = 0;
-        RWMutexScope(stripe->rw_mutex, write_mode)
-        {
-          for(TXT_Node *n = slot->first, *next = 0; n != 0; n = next)
-          {
-            next = n->next;
-            if(access_pt_is_expired(&n->access_pt) &&
-               n->load_count != 0 &&
-               n->is_working == 0)
-            {
-              slot_has_work = 1;
-              if(!write_mode)
               {
-                break;
+                U64 new_scope_num = scope_idx+1;
+                TXT_ScopeNode *parent = &shared->info.scope_nodes.v[top_scope_task->scope_idx];
+                if(parent->first_num == 0)
+                {
+                  parent->first_num = new_scope_num;
+                }
+                if(parent->last_num != 0)
+                {
+                  TXT_ScopeNode *prev_scope = &shared->info.scope_nodes.v[parent->last_num-1];
+                  prev_scope->next_num = new_scope_num;
+                }
+                parent->last_num = new_scope_num;
+                new_scope->parent_num = top_scope_task->scope_idx+1;
+              }
+              
+              // rjf: push onto scope stack
+              ScopeTask *scope_task = free_scope_task;
+              if(scope_task)
+              {
+                SLLStackPop(free_scope_task);
               }
               else
               {
-                DLLRemove(slot->first, slot->last, n);
+                scope_task = push_array(scratch.arena, ScopeTask, 1);
-                c_hash_downstream_dec(n->hash);
-                if(n->arena != 0)
-                {
-                  arena_release(n->arena);
-                }
-                SLLStackPush(txt_shared->stripes_free_nodes[(stripe - txt_shared->stripes.v)], n);
               }
+              scope_task->scope_idx = scope_idx;
+              scope_idx += 1;
+              SLLStackPush(top_scope_task, scope_task);
+            }
+            
+            // rjf: opener or closer -> fill endpoint
+            if(top_scope_task && (is_opener || is_closer))
+            {
+              shared->info.scope_pts.v[pt_idx].token_idx = token_idx;
+              shared->info.scope_pts.v[pt_idx].scope_idx = top_scope_task->scope_idx;
+              pt_idx += 1;
+            }
+            
+            // rjf: closer symbols -> pop
+            if(is_closer && top_scope_task != 0)
+            {
+              ScopeTask *popped = top_scope_task;
+              shared->info.scope_nodes.v[popped->scope_idx].token_idx_range.max = token_idx;
+              SLLStackPop(top_scope_task);
+              SLLStackPush(free_scope_task, popped);
             }
           }
         }
-        if(!slot_has_work)
+        scratch_end(scratch);
-        {
-          break;
-        }
       }
     }
+    lane_sync();
   }
+  
+  //- rjf: package as artifact
+  if(lane_idx() == 0 && shared->arena != 0)
+  {
+    shared->artifact = push_array(shared->arena, TXT_Artifact, 1);
+    shared->artifact->arena = shared->arena;
+    shared->artifact->info  = shared->info;
+  }
+  lane_sync();
+  
+  access_close(access);
+  scratch_end(scratch);
+  ProfEnd();
+  return shared->artifact;
+}
+
+internal void
+txt_artifact_destroy(void *ptr)
+{
+  if(ptr == 0) { return; }
+  TXT_Artifact *artifact = (TXT_Artifact *)ptr;
+  arena_release(artifact->arena);
+}
+
+internal TXT_TextInfo
+txt_text_info_from_hash_lang(Access *access, U128 hash, TXT_LangKind lang)
+{
+  struct
+  {
+    U128 hash;
+    TXT_LangKind lang;
+  } key = {hash, lang};
+  String8 key_string = str8_struct(&key);
+  TXT_Artifact *artifact = ac_artifact_from_key(access, key_string, txt_artifact_create, txt_artifact_destroy, 1024);
+  TXT_TextInfo info = {0};
+  if(artifact != 0)
+  {
+    info = artifact->info;
+  }
+  return info;
+}
+
+internal TXT_TextInfo
+txt_text_info_from_key_lang(Access *access, C_Key key, TXT_LangKind lang, U128 *hash_out)
+{
+  TXT_TextInfo result = {0};
+  for(U64 rewind_idx = 0; rewind_idx < C_KEY_HASH_HISTORY_COUNT; rewind_idx += 1)
+  {
+    U128 hash = c_hash_from_key(key, rewind_idx);
+    result = txt_text_info_from_hash_lang(access, hash, lang);
+    if(result.lines_count != 0)
+    {
+      if(hash_out)
+      {
+        *hash_out = hash;
+      }
+      break;
+    }
+  }
+  return result;
 }

src/text_cache/text_cache.h

@@ -208,9 +208,6 @@ struct TXT_Shared
 {
   Arena *arena;
   
-  // rjf: user clock
-  U64 user_clock_idx;
-  
   // rjf: cache
   U64 slots_count;
   TXT_Slot *slots;
@@ -240,7 +237,6 @@ struct TXT_Shared
 //~ rjf: Globals
 
 read_only global TXT_ScopeNode txt_scope_node_nil = {0};
-global TXT_Shared *txt_shared = 0;
 
 ////////////////////////////////
 //~ rjf: Basic Helpers
@@ -267,17 +263,6 @@ internal TXT_TokenArray txt_token_array_from_string__jai(Arena *arena, U64 *byte
 internal TXT_TokenArray txt_token_array_from_string__zig(Arena *arena, U64 *bytes_processed_counter, String8 string);
 internal TXT_TokenArray txt_token_array_from_string__disasm_x64_intel(Arena *arena, U64 *bytes_processed_counter, String8 string);
 
-////////////////////////////////
-//~ rjf: Main Layer Initialization
-
-internal void txt_init(void);
-
-////////////////////////////////
-//~ rjf: Cache Lookups
-
-internal TXT_TextInfo txt_text_info_from_hash_lang(Access *access, U128 hash, TXT_LangKind lang);
-internal TXT_TextInfo txt_text_info_from_key_lang(Access *access, C_Key key, TXT_LangKind lang, U128 *hash_out);
-
 ////////////////////////////////
 //~ rjf: Text Info Extractor Helpers
 
@@ -294,20 +279,11 @@ internal TXT_ScopeNode *txt_scope_node_from_info_off(TXT_TextInfo *info, U64 off
 internal TXT_ScopeNode *txt_scope_node_from_info_pt(TXT_TextInfo *info, TxtPt pt);
 
 ////////////////////////////////
-//~ rjf: Parse Threads
+//~ rjf: Artifact Cache Hooks / Lookups
 
-internal B32 txt_u2p_enqueue_req(U128 hash, TXT_LangKind lang, U64 endt_us);
+internal void *txt_artifact_create(String8 key);
-internal void txt_u2p_dequeue_req(U128 *hash_out, TXT_LangKind *lang_out);
+internal void txt_artifact_destroy(void *ptr);
-ASYNC_WORK_DEF(txt_parse_work);
+internal TXT_TextInfo txt_text_info_from_hash_lang(Access *access, U128 hash, TXT_LangKind lang);
-
+internal TXT_TextInfo txt_text_info_from_key_lang(Access *access, C_Key key, TXT_LangKind lang, U128 *hash_out);
-////////////////////////////////
-//~ rjf: Evictor Threads
-
-internal void txt_evictor_thread__entry_point(void *p);
-
-////////////////////////////////
-//~ rjf: Tick
-
-internal void txt_tick(void);
 
 #endif // TEXT_CACHE_H