eliminate old file stream cache, just use simplified cache for linting filesystem changes; otherwise switch everything to artifact cache

src/disasm/disasm.c

@@ -362,7 +362,7 @@ dasm_artifact_create(String8 key, B32 *retry_out)
                 {
                   // TODO(rjf): need redirection path - this may map to a different path on the local machine,
                   // need frontend to communicate path remapping info to this layer
-                  C_Key key = fs_key_from_path_range_new(file_normalized_full_path, r1u64(0, max_U64), 0);
+                  C_Key key = fs_key_from_path_range(file_normalized_full_path, r1u64(0, max_U64), 0);
                   TXT_LangKind lang_kind = txt_lang_kind_from_extension(file_normalized_full_path);
                   U64 endt_us = max_U64;
                   U128 hash = {0};

src/file_stream/file_stream.c

@@ -4,34 +4,6 @@
 #undef LAYER_COLOR
 #define LAYER_COLOR 0xfffa00ff
 
-////////////////////////////////
-//~ rjf: Basic Helpers
-
-internal U64
-fs_little_hash_from_string(String8 string)
-{
-  U64 result = 5381;
-  for(U64 i = 0; i < string.size; i += 1)
-  {
-    result = ((result << 5) + result) + string.str[i];
-  }
-  return result;
-}
-
-internal U128
-fs_big_hash_from_string_range(String8 string, Rng1U64 range)
-{
-  Temp scratch = scratch_begin(0, 0);
-  U64 buffer_size = string.size + sizeof(U64)*2;
-  U8 *buffer = push_array_no_zero(scratch.arena, U8, buffer_size);
-  MemoryCopy(buffer, string.str, string.size);
-  MemoryCopy(buffer + string.size, &range.min, sizeof(range.min));
-  MemoryCopy(buffer + string.size + sizeof(range.min), &range.max, sizeof(range.max));
-  U128 hash = c_hash_from_data(str8(buffer, buffer_size));
-  scratch_end(scratch);
-  return hash;
-}
-
 ////////////////////////////////
 //~ rjf: Top-Level API
 
@@ -43,17 +15,8 @@ fs_init(void)
   fs_shared->arena = arena;
   fs_shared->change_gen = 1;
   fs_shared->slots_count = 1024;
-  fs_shared->stripes_count = os_get_system_info()->logical_processor_count;
   fs_shared->slots = push_array(arena, FS_Slot, fs_shared->slots_count);
-  fs_shared->stripes = push_array(arena, FS_Stripe, fs_shared->stripes_count);
-  for(U64 idx = 0; idx < fs_shared->stripes_count; idx += 1)
-  {
-    fs_shared->stripes[idx].arena = arena_alloc();
-    fs_shared->stripes[idx].cv = cond_var_alloc();
-    fs_shared->stripes[idx].rw_mutex = rw_mutex_alloc();
-  }
-  fs_shared->req_mutex = mutex_alloc();
-  fs_shared->req_arena = arena_alloc();
+  fs_shared->stripes = stripe_array_alloc(arena);
 }
 
 ////////////////////////////////
@@ -68,6 +31,16 @@ fs_change_gen(void)
 ////////////////////////////////
 //~ rjf: Cache Interaction
 
+internal C_Key
+fs_content_key_from_artifact_key(String8 key)
+{
+  C_Key content_key = {0};
+  {
+    content_key.id.u128[0] = u128_hash_from_str8(key);
+  }
+  return content_key;
+}
+
 internal AC_Artifact
 fs_artifact_create(String8 key, B32 *retry_out)
 {
@@ -150,19 +123,17 @@ fs_artifact_create(String8 key, B32 *retry_out)
   }
   
   //- rjf: form content key
-  C_Key content_key = {0};
-  {
-    content_key.id.u128[0] = u128_hash_from_str8(key);
-  }
+  C_Key content_key = fs_content_key_from_artifact_key(key);
   
   //- rjf: abort if modification timestamps or sizes differ - we did not successfully read the file;
   //       otherwise submit data
+  B32 read_good = 0;
   if(lane_idx() == 0)
   {
-    B32 read_good = (pre_props.modified == post_props.modified &&
-                     pre_props.size == post_props.size &&
-                     data_buffer_size == total_bytes_read &&
-                     (file_handle_is_valid || pre_props.flags & FilePropertyFlag_IsFolder));
+    read_good = (pre_props.modified == post_props.modified &&
+                 pre_props.size == post_props.size &&
+                 data_buffer_size == total_bytes_read &&
+                 (file_handle_is_valid || pre_props.flags & FilePropertyFlag_IsFolder));
     if(!read_good)
     {
       retry_out[0] = 1;
@@ -182,6 +153,45 @@ fs_artifact_create(String8 key, B32 *retry_out)
   }
   lane_sync();
   
+  //- rjf: if the read was good, record this path's timestamp in this layer's path info cache
+  U64 path_hash = u64_hash_from_str8(path);
+  if(lane_idx() == 0 && read_good)
+  {
+    U64 slot_idx = path_hash%fs_shared->slots_count;
+    FS_Slot *slot = &fs_shared->slots[slot_idx];
+    Stripe *stripe = stripe_from_slot_idx(&fs_shared->stripes, slot_idx);
+    RWMutexScope(stripe->rw_mutex, 1)
+    {
+      FS_Node *node = 0;
+      for(FS_Node *n = slot->first; n != 0; n = n->next)
+      {
+        if(str8_match(n->path, path, 0))
+        {
+          node = n;
+          break;
+        }
+      }
+      if(node == 0)
+      {
+        node = stripe->free;
+        if(node)
+        {
+          stripe->free = node->next;
+        }
+        else
+        {
+          node = push_array_no_zero(stripe->arena, FS_Node, 1);
+        }
+        MemoryZeroStruct(node);
+        node->path = str8_copy(stripe->arena, path);
+        DLLPushBack(slot->first, slot->last, node);
+      }
+      node->last_modified_timestamp = pre_props.modified;
+      node->size = pre_props.size;
+    }
+  }
+  lane_sync();
+  
   //- rjf: bundle content key as artifact
   AC_Artifact artifact = {0};
   StaticAssert(sizeof(content_key) == sizeof(artifact), artifact_key_size_check);
@@ -197,11 +207,12 @@ fs_artifact_destroy(AC_Artifact artifact)
 {
   C_Key key = {0};
   MemoryCopyStruct(&key, &artifact);
+  key._padding_ = 0;
   c_close_key(key);
 }
 
 internal C_Key
-fs_key_from_path_range_new(String8 path, Rng1U64 range, U64 endt_us)
+fs_key_from_path_range(String8 path, Rng1U64 range, U64 endt_us)
 {
   C_Key result = {0};
   Temp scratch = scratch_begin(0, 0);
@@ -212,142 +223,37 @@ fs_key_from_path_range_new(String8 path, Rng1U64 range, U64 endt_us)
     str8_list_push(scratch.arena, &key_parts, path);
     str8_list_push(scratch.arena, &key_parts, str8_struct(&range));
     String8 key = str8_list_join(scratch.arena, &key_parts, 0);
-    AC_Artifact artifact = ac_artifact_from_key(access, key, fs_artifact_create, fs_artifact_destroy, endt_us);
+    
+    //- rjf: find generation number for this key
+    U64 gen = 0;
+    {
+      U64 hash = u64_hash_from_str8(path);
+      U64 slot_idx = hash%fs_shared->slots_count;
+      FS_Slot *slot = &fs_shared->slots[slot_idx];
+      Stripe *stripe = stripe_from_slot_idx(&fs_shared->stripes, slot_idx);
+      RWMutexScope(stripe->rw_mutex, 0)
+      {
+        for(FS_Node *n = slot->first; n != 0; n = n->next)
+        {
+          if(str8_match(path, n->path, 0))
+          {
+            gen = n->gen;
+            break;
+          }
+        }
+      }
+    }
+    
+    //- rjf: map to artifact
+    AC_Artifact artifact = ac_artifact_from_key(access, key, fs_artifact_create, fs_artifact_destroy, endt_us, .gen = gen);
     MemoryCopyStruct(&result, &artifact);
+    result._padding_ = 0;
   }
   access_close(access);
   scratch_end(scratch);
   return result;
 }
 
-internal C_Key
-fs_key_from_path_range(String8 path, Rng1U64 range, U64 endt_us)
-{
-  Temp scratch = scratch_begin(0, 0);
-  
-  //- rjf: unpack args
-  path = path_normalized_from_string(scratch.arena, path);
-  U64 path_little_hash = fs_little_hash_from_string(path);
-  U64 path_slot_idx = path_little_hash%fs_shared->slots_count;
-  U64 path_stripe_idx = path_slot_idx%fs_shared->stripes_count;
-  FS_Slot *path_slot = &fs_shared->slots[path_slot_idx];
-  FS_Stripe *path_stripe = &fs_shared->stripes[path_stripe_idx];
-  
-  //- rjf: get root for this path - on 1st try (read mode), try to read, on 2nd try (write mode), create node
-  C_Root root = {0};
-  for(B32 write_mode = 0; write_mode <= 1; write_mode += 1)
-  {
-    B32 node_found = 0;
-    RWMutexScope(path_stripe->rw_mutex, write_mode)
-    {
-      for(FS_Node *n = path_slot->first; n != 0; n = n->next)
-      {
-        if(str8_match(n->path, path, 0))
-        {
-          node_found = 1;
-          root = n->root;
-          break;
-        }
-      }
-      if(write_mode && !node_found)
-      {
-        FS_Node *node = push_array(path_stripe->arena, FS_Node, 1);
-        SLLQueuePush(path_slot->first, path_slot->last, node);
-        node->path = push_str8_copy(path_stripe->arena, path);
-        node->root = c_root_alloc();
-        node->slots_count = 64;
-        node->slots = push_array(path_stripe->arena, FS_RangeSlot, node->slots_count);
-        root = node->root;
-      }
-    }
-    if(node_found)
-    {
-      break;
-    }
-  }
-  
-  //- rjf: build a key for this path/range combo
-  C_Key key = c_key_make(root, c_id_make(range.min, range.max));
-  
-  //- rjf: if the most recent hash for this key is zero, then try to submit a new
-  // request to pull it in.
-  if(u128_match(c_hash_from_key(key, 0), u128_zero()))
-  {
-    // rjf: loop: request, check for results, return until we can't
-    RWMutexScope(path_stripe->rw_mutex, 1) for(;;)
-    {
-      // rjf: path -> node
-      FS_Node *node = 0;
-      for(FS_Node *n = path_slot->first; n != 0; n = n->next)
-      {
-        if(str8_match(path, n->path, 0))
-        {
-          node = n;
-          break;
-        }
-      }
-      
-      // rjf: no node? -> weird case, node should've been made at this point.
-      if(node == 0)
-      {
-        break;
-      }
-      
-      // rjf: range -> node
-      U64 range_hash = fs_little_hash_from_string(str8_struct(&key.id));
-      U64 range_slot_idx = range_hash%node->slots_count;
-      FS_RangeSlot *range_slot = &node->slots[range_slot_idx];
-      FS_RangeNode *range_node = 0;
-      for(FS_RangeNode *n = range_slot->first; n != 0; n = n->next)
-      {
-        if(c_id_match(n->id, key.id))
-        {
-          range_node = n;
-          break;
-        }
-      }
-      
-      // rjf: range node does not exist? create & store
-      if(range_node == 0)
-      {
-        range_node = push_array(path_stripe->arena, FS_RangeNode, 1);
-        SLLQueuePush(range_slot->first, range_slot->last, range_node);
-        range_node->id = key.id;
-      }
-      
-      // rjf: push request
-      if(range_node->working_count == 0)
-      {
-        range_node->working_count += 1;
-        MutexScope(fs_shared->req_mutex)
-        {
-          FS_RequestNode *n = push_array(fs_shared->req_arena, FS_RequestNode, 1);
-          SLLQueuePush(fs_shared->first_req, fs_shared->last_req, n);
-          fs_shared->req_count += 1;
-          n->v.key = key;
-          n->v.path = str8_copy(fs_shared->req_arena, path);
-          n->v.range = range;
-        }
-        cond_var_broadcast(async_tick_start_cond_var);
-      }
-      
-      // rjf: have time to wait? -> wait on this stripe; otherwise exit
-      B32 have_results = !u128_match(c_hash_from_key(key, 0), u128_zero());
-      if(!have_results && os_now_microseconds() < endt_us)
-      {
-        cond_var_wait_rw(path_stripe->cv, path_stripe->rw_mutex, 1, endt_us);
-      }
-      else
-      {
-        break;
-      }
-    }
-  }
-  
-  scratch_end(scratch);
-  return key;
-}
-
 internal U128
 fs_hash_from_path_range(String8 path, Rng1U64 range, U64 endt_us)
 {
@@ -372,8 +278,42 @@ fs_hash_from_path_range(String8 path, Rng1U64 range, U64 endt_us)
 internal void
 fs_async_tick(void)
 {
-#if 0
   ProfBeginFunction();
+  
+  //- rjf: detect changed timestamps for active paths
+  {
+    Rng1U64 range = lane_range(fs_shared->slots_count);
+    for EachInRange(slot_idx, range)
+    {
+      FS_Slot *slot = &fs_shared->slots[slot_idx];
+      Stripe *stripe = stripe_from_slot_idx(&fs_shared->stripes, slot_idx);
+      for(B32 write_mode = 0; write_mode <= 1; write_mode += 1)
+      {
+        B32 found_work = 0;
+        RWMutexScope(stripe->rw_mutex, write_mode)
+        {
+          for(FS_Node *n = slot->first; n != 0; n = n->next)
+          {
+            FileProperties props = os_properties_from_file_path(n->path);
+            if(props.modified != n->last_modified_timestamp)
+            {
+              found_work = 1;
+              if(write_mode)
+              {
+                n->gen += 1;
+              }
+            }
+          }
+        }
+        if(!found_work)
+        {
+          break;
+        }
+      }
+    }
+  }
+  
+#if 0
   Temp scratch = scratch_begin(0, 0);
   
   //- rjf: do detection pass
@@ -530,6 +470,6 @@ fs_async_tick(void)
   lane_sync();
   
   scratch_end(scratch);
-  ProfEnd();
 #endif
+  ProfEnd();
 }

src/file_stream/file_stream.h

@@ -5,38 +5,17 @@
 #define FILE_STREAM_H
 
 ////////////////////////////////
-//~ rjf: Per-Path Info Cache Types
-
-typedef struct FS_RangeNode FS_RangeNode;
-struct FS_RangeNode
-{
-  FS_RangeNode *next;
-  C_ID id;
-  U64 working_count;
-};
-
-typedef struct FS_RangeSlot FS_RangeSlot;
-struct FS_RangeSlot
-{
-  FS_RangeNode *first;
-  FS_RangeNode *last;
-};
+//~ rjf: Path Cache
 
 typedef struct FS_Node FS_Node;
 struct FS_Node
 {
   FS_Node *next;
-  
-  // rjf: file metadata
+  FS_Node *prev;
   String8 path;
-  FileProperties props;
-  
-  // rjf: hash store root
-  C_Root root;
-  
-  // rjf: sub-table of per-requested-file-range info
-  U64 slots_count;
-  FS_RangeSlot *slots;
+  U64 gen;
+  U64 last_modified_timestamp;
+  U64 size;
 };
 
 typedef struct FS_Slot FS_Slot;
@@ -46,54 +25,17 @@ struct FS_Slot
   FS_Node *last;
 };
 
-typedef struct FS_Stripe FS_Stripe;
-struct FS_Stripe
-{
-  Arena *arena;
-  CondVar cv;
-  RWMutex rw_mutex;
-};
-
 ////////////////////////////////
 //~ rjf: Shared State Bundle
 
-typedef struct FS_Request FS_Request;
-struct FS_Request
-{
-  FS_Request *next;
-  C_Key key;
-  String8 path;
-  Rng1U64 range;
-};
-
-typedef struct FS_RequestNode FS_RequestNode;
-struct FS_RequestNode
-{
-  FS_RequestNode *next;
-  FS_Request v;
-};
-
 typedef struct FS_Shared FS_Shared;
 struct FS_Shared
 {
   Arena *arena;
   U64 change_gen;
-  
-  // rjf: path info cache
   U64 slots_count;
-  U64 stripes_count;
   FS_Slot *slots;
-  FS_Stripe *stripes;
-  
-  // rjf: requests
-  Mutex req_mutex;
-  Arena *req_arena;
-  FS_RequestNode *first_req;
-  FS_RequestNode *last_req;
-  U64 req_count;
-  
-  // rjf: request take counter
-  U64 lane_req_take_counter;
+  StripeArray stripes;
 };
 
 ////////////////////////////////
@@ -101,12 +43,6 @@ struct FS_Shared
 
 global FS_Shared *fs_shared = 0;
 
-////////////////////////////////
-//~ rjf: Basic Helpers
-
-internal U64 fs_little_hash_from_string(String8 string);
-internal U128 fs_big_hash_from_string_range(String8 string, Rng1U64 range);
-
 ////////////////////////////////
 //~ rjf: Top-Level API
 
@@ -118,16 +54,17 @@ internal void fs_init(void);
 internal U64 fs_change_gen(void);
 
 ////////////////////////////////
-//~ rjf: Cache Interaction
+//~ rjf: Artifact Cache Hooks / Accessing API
+
+internal C_Key fs_content_key_from_artifact_key(String8 key);
 
 internal AC_Artifact fs_artifact_create(String8 key, B32 *retry_out);
 internal void fs_artifact_destroy(AC_Artifact artifact);
 
-internal C_Key fs_key_from_path_range_new(String8 path, Rng1U64 range, U64 endt_us);
-#define fs_key_from_path(path, endt_us) fs_key_from_path_range_new((path), r1u64(0, max_U64), (endt_us))
-
 internal C_Key fs_key_from_path_range(String8 path, Rng1U64 range, U64 endt_us);
 internal U128 fs_hash_from_path_range(String8 path, Rng1U64 range, U64 endt_us);
+#define fs_key_from_path(path, endt_us) fs_key_from_path_range((path), r1u64(0, max_U64), (endt_us))
+#define fs_hash_from_path(path, endt_us) fs_hash_from_path_range((path), r1u64(0, max_U64), (endt_us))
 
 ////////////////////////////////
 //~ rjf: Asynchronous Tick

src/raddbg/raddbg_core.c

@@ -1779,7 +1779,7 @@ rd_eval_space_read(void *u, E_Space space, void *out, Rng1U64 range)
       containing_range.max -= containing_range.max%chunk_size;
       
       // rjf: map to hash
-      C_Key key = fs_key_from_path_range_new(file_path, containing_range, 0);
+      C_Key key = fs_key_from_path_range(file_path, containing_range, 0);
       U128 hash = c_hash_from_key(key, 0);
       
       // rjf: look up from hash store
@@ -2153,7 +2153,7 @@ rd_key_from_eval_space_range(E_Space space, Rng1U64 range, B32 zero_terminated)
     {
       U64 file_path_string_id = space.u64_0;
       String8 file_path = e_string_from_id(file_path_string_id);
-      result = fs_key_from_path_range_new(file_path, range, 0);
+      result = fs_key_from_path_range(file_path, range, 0);
     }break;
     case RD_EvalSpaceKind_CtrlEntity:
     {