checkpoint on symbol server

project.raddbg

@@ -47,3 +47,9 @@ target:
   working_directory: build
   arguments: "--bin --quiet --rdi --out:C:/devel/raddebugger_stable/build/radbin.rdi --thread_count:1 C:/devel/raddebugger_stable/build/radbin.exe"
 }
+target:
+{
+  executable: "build/raddbg.exe"
+  working_directory: "build/"
+  arguments: "--bin --rdi C:\\SymbolCache\\ntdll.pdb\\180BF1B90AA75697D0EFEA5E5630AC7E1\\ntdll.pdb --thread_count=1"
+}

src/base/base_entry_point.c

@@ -224,6 +224,9 @@ async_thread_entry_point(void *params)
 #if defined(HTTP_H)
       http_async_tick();
 #endif
+#if defined(SYMBOL_SERVER_H)
+      smsv_async_tick();
+#endif
 #if defined(DBG_INFO_H)
       di_async_tick();
 #endif

src/base/base_thread_context.c

@@ -79,7 +79,6 @@ tctx_get_scratch(Arena **conflicts, U64 count)
       break;
     }
   }
-  Assert(result);
   return result;
 }
 

src/dbg_engine/dbg_engine_ctrl.c

@@ -2685,10 +2685,12 @@ d_ctrl_thread__module_open(D_Handle process, D_Handle module, U64 base_vaddr, DM
   //////////////////////////////
   //- rjf: no found debug info path -> try to fall back on symbol server
   //
+#if 0
   if(initial_debug_info_path.size == 0)
   {
-    initial_debug_info_path = smsv_local_path_from_key(arena, str8_skip_last_slash(module_info->module_path), str8_skip_last_slash(module_info->debug_info_path), module_info->debug_info_guid, module_info->debug_info_age);
+    initial_debug_info_path = smsv_local_path_from_key(arena, str8_skip_last_slash(module_info->debug_info_path), module_info->debug_info_guid, module_info->debug_info_age);
   }
+#endif
   
   //////////////////////////////
   //- rjf: write 1 at attachment marker, to signify attachment

src/dbg_info/dbg_info.c

@@ -680,8 +680,22 @@ di_async_tick(void)
           }
         }
         
+        //- rjf: check with the symbol server to see if we are actively downloading O.G.
+        B32 og_is_downloading = 0;
+        {
+          B32 file_is_present = (properties_from_file_path(og_path).modified != 0);
+          if(!file_is_present)
+          {
+            SMSV_Status status = smsv_status_from_local_path(og_path);
+            if(status == SMSV_Status_Pending)
+            {
+              og_is_downloading = 1;
+            }
+          }
+        }
+        
         //- rjf: analyze O.G. debug info
-        if(!t->og_analyzed)
+        if(!t->og_analyzed && !og_is_downloading)
         {
           t->og_analyzed = 1;
           File file = file_open(AccessFlag_ShareRead|AccessFlag_Read, og_path);
@@ -714,7 +728,7 @@ di_async_tick(void)
         }
         
         //- rjf: determine if RDI is stale
-        if(!t->rdi_analyzed)
+        if(!t->rdi_analyzed && !og_is_downloading)
         {
           t->rdi_analyzed = 1;
           File file = file_open(AccessFlag_ShareRead|AccessFlag_Read, rdi_path);
@@ -737,7 +751,7 @@ di_async_tick(void)
         B32 rdi_is_stale = t->rdi_is_stale;
         
         //- rjf: calculate thread counts for conversion processes
-        if(!og_is_rdi && rdi_is_stale && t->thread_count == 0)
+        if(!og_is_rdi && rdi_is_stale && t->thread_count == 0 && !og_is_downloading)
         {
           U64 thread_count = 1;
           U64 max_thread_count = get_system_info()->logical_processor_count/2;
@@ -768,7 +782,7 @@ di_async_tick(void)
         
         //- rjf: if this conversion will overwrite an RDI we already have in cache,
         // then we need to evict the old one from the cache.
-        B32 ready_to_launch_conversion = (threads_available && !og_is_rdi && rdi_is_stale && t->thread_count != 0 && t->status != DI_LoadTaskStatus_Active);
+        B32 ready_to_launch_conversion = (!og_is_downloading && threads_available && !og_is_rdi && rdi_is_stale && t->thread_count != 0 && t->status != DI_LoadTaskStatus_Active);
         if(ready_to_launch_conversion)
         {
           U64 path2key_hash = u64_hash_from_str8(og_path);
@@ -863,7 +877,7 @@ di_async_tick(void)
         
         //- rjf: if the RDI for this task is not stale, then we're already done - mark this
         // task as done & prepped for storing into the cache
-        if(!rdi_is_stale)
+        if(!rdi_is_stale && !og_is_downloading)
         {
           t->status = DI_LoadTaskStatus_Done;
         }

src/dbg_info/dbg_info.h

@@ -140,6 +140,7 @@ struct DI_LoadTask
   B32 og_analyzed;
   B32 og_is_rdi;
   U64 og_size;
+  B32 og_is_downloading;
   
   B32 rdi_analyzed;
   B32 rdi_is_stale;

src/http/http.c

@@ -31,3 +31,21 @@ http_status_kind_from_code(HTTP_StatusCode code)
   }
   return kind;
 }
+
+////////////////////////////////////////////////////////////////
+//~ rjf: Synchronous Request Helper
+
+internal HTTP_Response
+http_request(Arena *arena, HTTP_RequestParams *params)
+{
+  HTTP_Response response = {0};
+  Temp scratch = scratch_begin(&arena, 1);
+  GuardedRing *gr = guarded_ring_alloc(scratch.arena, KB(64));
+  {
+    http_push_request(gr, params, max_U64);
+    http_pop_response(arena, gr, &response, max_U64);
+  }
+  guarded_ring_release(gr);
+  scratch_end(scratch);
+  return response;
+}

src/http/http.h

@@ -146,6 +146,7 @@ typedef struct HTTP_Response HTTP_Response;
 struct HTTP_Response
 {
   U64 id;
+  B32 has_more;
   HTTP_StatusCode code;
   String8 body;
 };
@@ -155,6 +156,11 @@ struct HTTP_Response
 
 internal HTTP_StatusKind http_status_kind_from_code(HTTP_StatusCode code);
 
+////////////////////////////////////////////////////////////////
+//~ rjf: Synchronous Request Helper
+
+internal HTTP_Response http_request(Arena *arena, HTTP_RequestParams *params);
+
 ////////////////////////////////////////////////////////////////
 //~ rjf: @per_os_impl Top-Level Layer Calls
 

src/pdb/pdb_parse.c

@@ -5,17 +5,21 @@
 //~ PDB Parser Functions
 
 internal PDB_Info*
-pdb_info_from_data(Arena *arena, String8 data){
+pdb_info_from_data(Arena *arena, String8 data)
+{
   ProfBegin("pdb_info_from_data");
+  PDB_Info *result = push_array(arena, PDB_Info, 1);
   
   // get header
   PDB_InfoHeader *header = 0;
-  if (data.size >= sizeof(*header)){
+  if(data.size >= sizeof(*header))
+  {
     header = (PDB_InfoHeader *)data.str;
   }
   
-  PDB_Info *result = 0;
+  // rjf: parse info given header
-  if (header != 0){
+  if(header != 0)
+  {
     // read guid
     Guid *auth_guid = 0;
     U32 after_auth_guid_off = sizeof(*header);
@@ -122,7 +126,6 @@ pdb_info_from_data(Arena *arena, String8 data){
   }
   
   ProfEnd();
-  
   return(result);
 }
 
@@ -757,6 +760,37 @@ pdb_comp_unit_contribution_array_from_data(Arena *arena, String8 data, COFF_Sect
   return result;
 }
 
+internal PDB_CompUnitContribution *
+pdb_comp_unit_contribution_from_voff__binary_search(PDB_CompUnitContributionArray *array, U64 voff)
+{
+  PDB_CompUnitContribution *result = 0;
+  if(array->count != 0)
+  {
+    U64 first_idx = 0;
+    U64 last_idx = array->count-1;
+    for(;first_idx < last_idx;)
+    {
+      U64 mid_idx = (last_idx + first_idx) / 2;
+      U64 mid_voff_first = array->contributions[mid_idx].voff_first;
+      U64 mid_voff_opl = array->contributions[mid_idx].voff_opl;
+      if(voff < mid_voff_first)
+      {
+        last_idx = mid_idx;
+      }
+      else if(mid_voff_opl <= voff)
+      {
+        first_idx = mid_idx+1;
+      }
+      else
+      {
+        result = &array->contributions[mid_idx];
+        break;
+      }
+    }
+  }
+  return result;
+}
+
 ////////////////////////////////
 //~ PDB Dbi Functions
 

src/pdb/pdb_parse.h

@@ -212,6 +212,7 @@ internal PDB_CompUnitArray*   pdb_comp_unit_array_from_data(Arena *arena,
                                                             String8 module_info_data);
 
 internal PDB_CompUnitContributionArray pdb_comp_unit_contribution_array_from_data(Arena *arena, String8 seccontrib_data, COFF_SectionHeaderArray sections);
+internal PDB_CompUnitContribution *pdb_comp_unit_contribution_from_voff__binary_search(PDB_CompUnitContributionArray *array, U64 voff);
 
 ////////////////////////////////
 //~ PDB Dbi Functions

src/raddbg/raddbg_core.c

@@ -10684,7 +10684,8 @@ rd_init(CmdLine *cmdln)
   
   // rjf: set up update check http ring & send request
   {
-    rd_state->update_check_http_ring = guarded_ring_alloc(arena, KB(8));
+    rd_state->update_check_arena = arena_alloc();
+    rd_state->update_check_http_ring = guarded_ring_alloc(rd_state->update_check_arena, KB(8));
     HTTP_RequestParams p =
     {
       .id = 1,
@@ -11102,12 +11103,24 @@ rd_frame(void)
   //
   if(!rd_state->got_update_check)
   {
-    Temp scratch = scratch_begin(0, 0);
+    // rjf: pop all responses that we can
-    HTTP_Response response = {0};
+    B32 done = 0;
-    if(http_pop_response(scratch.arena, rd_state->update_check_http_ring, &response, 0))
+    for(HTTP_Response response = {0}; http_pop_response(rd_state->update_check_arena, rd_state->update_check_http_ring, &response, 0);)
+    {
+      str8_list_push(rd_state->update_check_arena, &rd_state->update_check_response_body_pieces, response.body);
+      if(response.has_more == 0)
+      {
+        done = 1;
+      }
+    }
+    
+    // rjf: if we're done -> parse response & determine if there is a new version
+    if(done)
     {
       rd_state->got_update_check = 1;
-      MD_Node *root = md_tree_from_string(scratch.arena, response.body)->first;
+      Temp scratch = scratch_begin(0, 0);
+      String8 response_body = str8_list_join(rd_state->update_check_arena, &rd_state->update_check_response_body_pieces, 0);
+      MD_Node *root = md_tree_from_string(scratch.arena, response_body)->first;
       MD_Node *html_url = md_child_from_string(root, s("html_url"), 0);
       String8 newest_release_url = html_url->first->string;
       U64 tag_pos = str8_find_needle(newest_release_url, 0, s("tag/v"), 0);
@@ -11120,9 +11133,14 @@ rd_frame(void)
       {
         rd_state->newer_update_available = 1;
       }
-    }
+      guarded_ring_release(rd_state->update_check_http_ring);
+      arena_release(rd_state->update_check_arena);
+      rd_state->update_check_arena = 0;
+      rd_state->update_check_http_ring = 0;
+      MemoryZeroStruct(&rd_state->update_check_response_body_pieces);
       scratch_end(scratch);
     }
+  }
   
   //////////////////////////////
   //- rjf: iterate all tabs, touch their view-states
@@ -17380,7 +17398,9 @@ rd_frame(void)
           D_Entity *module = d_entity_from_handle(evt->entity);
           D_Entity *debug_info_path = d_entity_child_from_kind(module, D_EntityKind_DebugInfoPath);
           String8 new_path = debug_info_path->string;
-          if(new_path.size != 0 && file_path_exists(new_path))
+          B32 file_exists = file_path_exists(new_path);
+          B32 file_is_pending = (!file_exists ? smsv_status_from_local_path(new_path) == SMSV_Status_Pending : 0);
+          if(new_path.size != 0 && (file_exists || file_is_pending))
           {
             CFG_NodePtrList dbg_infos = cfg_node_top_level_list_from_string(scratch.arena, str8_lit("debug_info"));
             B32 path_found = 0;

src/raddbg/raddbg_core.h

@@ -397,6 +397,8 @@ struct RD_State
   // rjf: update checker
   B32 got_update_check;
   B32 newer_update_available;
+  Arena *update_check_arena;
+  String8List update_check_response_body_pieces;
   GuardedRing *update_check_http_ring;
   
   // rjf: config bucket paths

src/raddbg/raddbg_main.c

@@ -5,6 +5,7 @@
 //~ rjf: post-0.9.26 TODO notes
 //
 // [ ] unwinding crash - f5 on bp in w32_http_status_callback to repro
+// [ ] unwinding in jit page - fail
 // [ ] jeff stepping bug
 // [ ] wassim memory bug
 // [ ] switch off spoofs for step-over
@@ -514,6 +515,34 @@ entry_point(CmdLine *cmd_line)
 {
   Temp scratch = scratch_begin(0, 0);
   
+  //- TODO(rjf): test symbol server download
+#if 0
+  {
+    GuardedRing *out_ring = guarded_ring_alloc(scratch.arena, KB(64));
+    {
+      HTTP_RequestParams p =
+      {
+        .id = 1,
+        .method = HTTP_Method_Get,
+        .url = s("https://msdl.microsoft.com/download/symbols/ntdll.pdb/E84AF4A1E3CDFABC99A41209581EB6861/ntdll.pdb"),
+      };
+      http_push_request(out_ring, &p, max_U64);
+      String8List pieces = {0};
+      for(HTTP_Response r = {0}; http_pop_response(scratch.arena, out_ring, &r, max_U64);)
+      {
+        str8_list_push(scratch.arena, &pieces, r.body);
+        if(r.has_more == 0)
+        {
+          int x = 0;
+          break;
+        }
+      }
+      int x = 0;
+    }
+    guarded_ring_release(out_ring);
+  }
+#endif
+  
   //- rjf: unpack command line arguments
   ExecMode exec_mode = ExecMode_Normal;
   B32 auto_run = 0;

src/rdi_from_pdb/rdi_from_pdb.c

@@ -397,22 +397,18 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
 {
   Temp scratch = scratch_begin(&arena, 1);
   
-  Arena *scratch_conflicts[2] = { scratch.arena, arena };
-  
   //////////////////////////////////////////////////////////////
   //- rjf: do base MSF parse
   //
   MSF_Parsed *msf = 0;
   ProfScope("do base MSF parse")
   {
-    Temp scratch2 = scratch_begin(scratch_conflicts, ArrayCount(scratch_conflicts));
-    
     // rjf: setup output buckets
     MSF_RawStreamTable *msf_raw_stream_table = 0;
     U64 *msf_stream_take_counter = 0;
     if(lane_idx() == 0)
     {
-      msf_raw_stream_table = msf_raw_stream_table_from_data(scratch2.arena, params->input_pdb_data);
+      msf_raw_stream_table = msf_raw_stream_table_from_data(scratch.arena, params->input_pdb_data);
       msf = push_array(scratch.arena, MSF_Parsed, 1);
       msf->page_size = msf_raw_stream_table->page_size;
       msf->page_count = msf_raw_stream_table->total_page_count;
@@ -437,8 +433,6 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
       }
     }
     lane_sync();
-    
-    scratch_end(scratch2);
   }
   lane_sync();
   
@@ -677,7 +671,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
   //
   U64 exe_voff_max = 0;
   {
-    Temp scratch2 = scratch_begin(scratch_conflicts, ArrayCount(scratch_conflicts));
+    Temp scratch2 = scratch_begin(&scratch.arena, 1);
     
     // rjf: set up
     U64 *lane_voff_maxes = 0;
@@ -827,7 +821,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
     //- rjf: do wide gather
     ProfScope("do wide gather")
     {
-      Temp scratch2 = scratch_begin(scratch_conflicts, ArrayCount(scratch_conflicts));
+      Temp scratch2 = scratch_begin(&scratch.arena, 1);
       
       //- rjf: build local hash table to dedup files within this lane
       U64 hit_path_slots_count = 4096;
@@ -1758,7 +1752,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
         }
         
         //- rjf: walk dependent types, push to chain
-        Temp scratch2 = scratch_begin(scratch_conflicts, ArrayCount(scratch_conflicts));
+        Temp scratch2 = scratch_begin(&scratch.arena, 1);
         P2R_TypeIdChain start_walk_task = {0, itype};
         P2R_TypeIdChain *first_walk_task = &start_walk_task;
         P2R_TypeIdChain *last_walk_task = &start_walk_task;
@@ -1767,7 +1761,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
             walk_task = walk_task->next)
         {
           CV_TypeId walk_itype = itype_fwd_map[walk_task->itype] ? itype_fwd_map[walk_task->itype] : walk_task->itype;
-          if(walk_itype < tpi_leaf->itype_first)
+          if(walk_itype < tpi_leaf->itype_first || tpi_leaf->itype_opl <= walk_itype)
           {
             continue;
           }
@@ -2041,7 +2035,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
   U64 all_namespace_slots_count = 0;
   ProfScope("gather all unique namespaces from types")
   {
-    Temp scratch2 = scratch_begin(scratch_conflicts, ArrayCount(scratch_conflicts));
+    Temp scratch2 = scratch_begin(&scratch.arena, 1);
     
     //- rjf: find all unique namespaces on this lane
     U64 namespace_slots_count = (U64)itype_opl / lane_count();
@@ -2874,7 +2868,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
                              RDIM_SubsetFlag_Types))
     ProfScope("gather all namespaces which are only encoded in symbols (not found in types)")
   {
-    Temp scratch2 = scratch_begin(scratch_conflicts, ArrayCount(scratch_conflicts));
+    Temp scratch2 = scratch_begin(&scratch.arena, 1);
     U64 lane_namespace_slots_count = 4096;
     String8Node **lane_namespace_slots = push_array(scratch2.arena, String8Node *, lane_namespace_slots_count);
     
@@ -3537,6 +3531,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
                     if(n->voff == voff)
                     {
                       link_name = n->name;
+                      ins_atomic_u64_inc_eval(&n->referencing_symbol_count);
                       break;
                     }
                   }
@@ -4183,6 +4178,82 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
   }
   lane_sync();
   
+  //////////////////////////////////////////////////////////////
+  //- rjf: produce stub procedures from all pub32 symbols
+  //
+  if(lane_idx() == 0)
+  {
+    CV_SymParsed *sym = all_syms[0];
+    for(CV_RecIter iter = {0}; cv_rec_next(sym->data, &sym->sym_ranges, 0, &iter);)
+    {
+      RDIM_Unit *sym_unit = &all_units_ptr->first->v[0];
+      RDIM_SymbolChunkList *sym_procedures = &sym_unit->procedures;
+      RDIM_SymbolChunkList *sym_global_variables = &sym_unit->global_variables;
+      RDIM_ScopeChunkList *sym_scopes = &sym_unit->scopes;
+      switch(iter.kind)
+      {
+        default:{}break;
+        case CV_SymKind_PUB32:
+        {
+          // rjf: unpack record
+          CV_SymPub32 *pub32 = (CV_SymPub32 *)iter.struct_base;
+          String8 name = str8_cstring_capped(pub32+1, iter.opl);
+          COFF_SectionHeader *section = (0 < pub32->sec && pub32->sec <= coff_sections.count) ? &coff_sections.v[pub32->sec-1] : 0;
+          U64 voff = 0;
+          if(section != 0)
+          {
+            voff = section->voff + pub32->off;
+          }
+          
+          // rjf: determine if this pub32 was covered by another symbol record
+          B32 was_in_other_record = 0;
+          {
+            U64 hash = p2r_hash_from_voff(voff);
+            U64 slot_idx = hash%link_name_map->buckets_count;
+            for(P2R_LinkNameNode *n = link_name_map->buckets[slot_idx]; n != 0; n = n->next)
+            {
+              if(n->voff == voff)
+              {
+                was_in_other_record = (n->referencing_symbol_count != 0);
+                break;
+              }
+            }
+          }
+          
+          // rjf: find compilation unit contribution to determine pub32's size
+          U64 voff_opl = voff+1;
+          if(!was_in_other_record)
+          {
+            PDB_CompUnitContribution *contrib = pdb_comp_unit_contribution_from_voff__binary_search(comp_unit_contributions, voff);
+            if(contrib != 0)
+            {
+              voff_opl = contrib->voff_opl;
+            }
+          }
+          
+          // rjf: if this pub32 was *not* covered by another symbol record, build a symbol for it
+          if(!was_in_other_record)
+          {
+            B32 is_procedure = (pub32->flags & (CV_Pub32Flag_Code|CV_Pub32Flag_Function));
+            RDIM_SymbolChunkList *symbols = is_procedure ? sym_procedures : sym_global_variables;
+            RDIM_Symbol *symbol = rdim_symbol_chunk_list_push(arena, symbols, 64);
+            symbol->name = name;
+            symbol->link_name = name;
+            if(is_procedure)
+            {
+              RDIM_Scope *scope = rdim_scope_chunk_list_push(arena, sym_scopes, 64);
+              scope->symbol = symbol;
+              RDIM_Rng1U64 range = {voff, voff_opl};
+              rdim_scope_push_voff_range(arena, sym_scopes, scope, range);
+              symbol->root_scope = scope;
+            }
+          }
+        }break;
+      }
+    }
+  }
+  lane_sync();
+  
   //////////////////////////////////////////////////////////////
   //- rjf: upgrade namespace nodes with scopes info, if they match a scope
   //

src/rdi_from_pdb/rdi_from_pdb.h

@@ -29,6 +29,7 @@ struct P2R_LinkNameNode
   P2R_LinkNameNode *next;
   U64 voff;
   String8 name;
+  U64 referencing_symbol_count;
 };
 
 typedef struct P2R_LinkNameMap P2R_LinkNameMap;

src/symbol_server/symbol_server.h

@@ -4,7 +4,16 @@
 #ifndef SYMBOL_SERVER_H
 #define SYMBOL_SERVER_H
 
+typedef enum SMSV_Status
+{
+  SMSV_Status_Null,
+  SMSV_Status_Pending,
+}
+SMSV_Status;
+
 internal void smsv_init(void);
-internal String8 smsv_local_path_from_key(Arena *arena, String8 module_name, String8 dbg_name, Guid guid, U64 age);
+internal void smsv_async_tick(void);
+internal String8 smsv_local_path_from_key(Arena *arena, String8 dbg_name, Guid guid, U64 age);
+internal SMSV_Status smsv_status_from_local_path(String8 path);
 
 #endif // SYMBOL_SERVER_H

src/symbol_server/symbol_server_stub.c

@@ -6,8 +6,20 @@ smsv_init(void)
 {
 }
 
+internal void
+smsv_async_tick(void)
+{
+}
+
 internal String8
-smsv_local_path_from_key(Arena *arena, String8 module_name, String8 dbg_name, Guid guid, U64 age)
+smsv_local_path_from_key(Arena *arena, String8 dbg_name, Guid guid, U64 age)
 {
   return s("");
 }
+
+internal SMSV_Status
+smsv_status_from_local_path(String8 path)
+{
+  SMSV_Status status = SMSV_Status_Null;
+  return status;
+}

src/win32/http/win32_http.c

@@ -293,6 +293,7 @@ http_async_tick(void)
         }
         req->hConnect = hConnect;
         req->hRequest = hRequest;
+        req->arena_start_body_read_pos = arena_pos(req->arena);
         
         //- rjf: send request
         if(good)
@@ -324,6 +325,95 @@ http_async_tick(void)
     }
   }
   
+  //- rjf: push response output
+  {
+    Rng1U64 range = lane_range(ArrayCount(w32_http_state->active_requests));
+    for EachInRange(idx, range)
+    {
+      W32_HTTP_Request *req = &w32_http_state->active_requests[idx];
+      W32_HTTP_RequestStatus status = ins_atomic_u32_eval(&req->status);
+      if(req->active && (status == W32_HTTP_RequestStatus_Done ||
+                         status == W32_HTTP_RequestStatus_Failed ||
+                         req->finished_body_pieces.total_size >= req->out_ring->ring->size/2))
+      {
+        Temp scratch = scratch_begin(0, 0);
+        
+        // rjf: figure out status code
+        if(req->status_code == 0)
+        {
+          DWORD status_code = 0;
+          DWORD status_code_size = sizeof(status_code);
+          WinHttpQueryHeaders(req->hRequest, 
+                              WINHTTP_QUERY_STATUS_CODE|WINHTTP_QUERY_FLAG_NUMBER, 
+                              WINHTTP_HEADER_NAME_BY_INDEX, 
+                              &status_code, &status_code_size, WINHTTP_NO_HEADER_INDEX);
+          req->status_code = status_code;
+        }
+        
+        // rjf: join all currently read body contents
+        String8 body = str8_list_join(scratch.arena, &req->finished_body_pieces, 0);
+        
+        // rjf: clear current body output
+        arena_pop_to(req->arena, req->arena_start_body_read_pos);
+        MemoryZeroStruct(&req->finished_body_pieces);
+        
+        // rjf: compute size per record header
+        U64 max_record_size = req->out_ring->ring->size;
+        U64 bytes_per_record_header = sizeof(U64) + sizeof(U64) + sizeof(B32) + sizeof(HTTP_StatusCode);
+        
+        // rjf: push response records
+        for(U64 off = 0, next_off = 0; off <= body.size; off = next_off)
+        {
+          // rjf: determine how much of the body we can fit into this record
+          U64 body_data_size = (body.size - off);
+          if(bytes_per_record_header + body_data_size > max_record_size)
+          {
+            body_data_size = max_record_size - bytes_per_record_header;
+          }
+          
+          // rjf: advance by that amount of the body
+          next_off = off + body_data_size;
+          
+          // rjf: serialize this record
+          String8 record_data = {0};
+          {
+            String8List pieces = {0};
+            U64 record_size = bytes_per_record_header + (next_off - off);
+            B32 is_last = (next_off == body.size && (status == W32_HTTP_RequestStatus_Done || status == W32_HTTP_RequestStatus_Failed));
+            B32 has_more = !is_last;
+            str8_list_push(scratch.arena, &pieces, str8_struct(&record_size));
+            str8_list_push(scratch.arena, &pieces, str8_struct(&req->id));
+            str8_list_push(scratch.arena, &pieces, str8_struct(&has_more));
+            str8_list_push(scratch.arena, &pieces, str8_struct(&req->status_code));
+            str8_list_push(scratch.arena, &pieces, str8_substr(body, r1u64(off, next_off)));
+            record_data = str8_list_join(scratch.arena, &pieces, 0);
+          }
+          
+          // rjf: push this record
+          {
+            // TODO(rjf): can't wait here technically, because we might be waiting for other
+            // work in the async threads to pop! so we'll just deadlock right here. two options:
+            //
+            // (a) http requests need to be on another timeline
+            // (b) we need to prepare for this to fail & batch it up or something.
+            //
+            RingGuard g = guarded_ring_open(req->out_ring);
+            guarded_ring_write_string_or_wait(&g, record_data, max_U64);
+            guarded_ring_close(&g);
+          }
+          
+          // rjf: cancel on no movement
+          if(next_off == off)
+          {
+            break;
+          }
+        }
+        
+        scratch_end(scratch);
+      }
+    }
+  }
+  
   //- rjf: kick off body size queries
   {
     Rng1U64 range = lane_range(ArrayCount(w32_http_state->active_requests));
@@ -364,62 +454,10 @@ http_async_tick(void)
     {
       W32_HTTP_Request *req = &w32_http_state->active_requests[idx];
       W32_HTTP_RequestStatus status = ins_atomic_u32_eval(&req->status);
-      if(req->active &&
+      if(req->active && req->finished_body_pieces.total_size == 0 && req->next_body_piece_size == 0 &&
          (status == W32_HTTP_RequestStatus_Done ||
           status == W32_HTTP_RequestStatus_Failed))
       {
-        Temp scratch = scratch_begin(0, 0);
-        B32 good = (status == W32_HTTP_RequestStatus_Done);
-        
-        //- rjf: unpack
-        HINTERNET hRequest = req->hRequest;
-        
-        //- rjf: read response code
-        HTTP_StatusCode code = 0;
-        if(good)
-        {
-          DWORD status_code = 0;
-          DWORD status_code_size = sizeof(status_code);
-          WinHttpQueryHeaders(hRequest, 
-                              WINHTTP_QUERY_STATUS_CODE|WINHTTP_QUERY_FLAG_NUMBER, 
-                              WINHTTP_HEADER_NAME_BY_INDEX, 
-                              &status_code, &status_code_size, WINHTTP_NO_HEADER_INDEX);
-          code = status_code;
-        }
-        
-        //- rjf: serialize response
-        String8 response_data = {0};
-        U64 response_data_min_size = sizeof(U64) + sizeof(U64) + sizeof(HTTP_StatusCode) + sizeof(U64);
-        U64 response_data_max_size = req->out_ring->ring->size;
-        B32 response_is_possible = (response_data_min_size < response_data_max_size);
-        if(response_is_possible)
-        {
-          U64 body_size = req->finished_body_pieces.total_size;
-          String8List pieces = {0};
-          str8_list_push(scratch.arena, &pieces, str8_struct(&req->id));
-          str8_list_push(scratch.arena, &pieces, str8_struct(&code));
-          str8_list_push(scratch.arena, &pieces, str8_struct(&body_size));
-          str8_list_concat_in_place(&pieces, &req->finished_body_pieces);
-          U64 data_size = pieces.total_size;
-          str8_list_push_front(scratch.arena, &pieces, str8_struct(&data_size));
-          if(data_size + sizeof(U64) > response_data_max_size)
-          {
-            U64 overkill = data_size + sizeof(U64) - response_data_max_size;
-            data_size -= overkill;
-          }
-          response_data = str8_list_join(scratch.arena, &pieces, 0);
-          response_data.size = data_size + sizeof(U64);
-        }
-        
-        //- rjf: write response
-        if(response_is_possible)
-        {
-          RingGuard g = guarded_ring_open(req->out_ring);
-          guarded_ring_write_string_or_wait(&g, response_data, max_U64);
-          guarded_ring_close(&g);
-        }
-        
-        //- rjf: free this request slot
         req->active = 0;
         arena_clear(req->arena);
         MemoryZeroStruct(&req->finished_body_pieces);
@@ -427,8 +465,6 @@ http_async_tick(void)
         req->next_body_piece_size = 0;
         WinHttpCloseHandle(req->hConnect);
         WinHttpCloseHandle(req->hRequest);
-        
-        scratch_end(scratch);
       }
     }
   }
@@ -485,14 +521,16 @@ http_pop_response(Arena *arena, GuardedRing *out_ring, HTTP_Response *response_o
   B32 result = guarded_ring_read_struct_or_wait(&g, &data_size, endt_us);
   if(result)
   {
+    U64 data_size_minus_size_prefix = data_size - sizeof(data_size);
     String8 data = {0};
-    data.size = data_size;
+    data.size = data_size_minus_size_prefix;
     data.str = push_array(arena, U8, data.size);
     guarded_ring_read_or_wait(&g, data.size, data.str, max_U64);
     U64 off = 0;
     off += str8_deserial_read_struct(data, off, &response_out->id);
+    off += str8_deserial_read_struct(data, off, &response_out->has_more);
     off += str8_deserial_read_struct(data, off, &response_out->code);
-    off += str8_deserial_read_struct(data, off, &response_out->body.size);
+    response_out->body.size = (data.size - off);
     response_out->body.str = push_array(arena, U8, response_out->body.size);
     off += str8_deserial_read(data, off, response_out->body.str, response_out->body.size, 1);
   }

src/win32/http/win32_http.h

@@ -34,6 +34,8 @@ struct W32_HTTP_Request
   HINTERNET hRequest;
   void *optional; // NOTE(rjf): must persist with in-flight requests
   U64 optional_size;
+  HTTP_StatusCode status_code;
+  U64 arena_start_body_read_pos;
   String8List finished_body_pieces;
   void *next_body_piece;
   U64 next_body_piece_size;

src/win32/symbol_server/win32_symbol_server.c

@@ -7,6 +7,8 @@ smsv_init(void)
   Arena *arena = arena_alloc();
   w32_smsv_state = push_array(arena, W32_SMSV_State, 1);
   w32_smsv_state->arena = arena;
+  
+  // rjf: get symbol server configuration from environment
   {
     // rjf: extract parameterizations from _NT_SYMBOL_PATH
     {
@@ -71,24 +73,257 @@ smsv_init(void)
       }
     }
     
-    // rjf fall back to using local appdata temp symbol cache folder
+    // rjf fall back to using C:/SymbolCache (or wherever windows is located)
     if(w32_smsv_state->symbol_cache_path.size == 0)
     {
       Temp scratch = scratch_begin(0, 0);
       U64 size = KB(32);
       U16 *buffer = push_array_no_zero(scratch.arena, U16, size);
-      if(SUCCEEDED(SHGetFolderPathW(0, CSIDL_LOCAL_APPDATA, 0, 0, (WCHAR*)buffer)))
+      if(SUCCEEDED(SHGetFolderPathW(0, CSIDL_SYSTEM, 0, 0, (WCHAR*)buffer)))
       {
-        String8 local_appdata = str8_from_16(scratch.arena, str16_cstring(buffer));
+        String8 system32 = str8_from_16(scratch.arena, str16_cstring(buffer));
-        w32_smsv_state->symbol_cache_path = str8f(arena, "%S\\Temp\\SymbolCache", local_appdata);
+        String8 containing_drive = str8_prefix(system32, str8_find_needle(system32, 0, s("/"), StringMatchFlag_SlashInsensitive));
+        w32_smsv_state->symbol_cache_path = str8f(arena, "%S\\SymbolCache", containing_drive);
       }
       scratch_end(scratch);
     }
   }
+  
+  // rjf: set up task cache
+  {
+    w32_smsv_state->task_slots_count = 256;
+    w32_smsv_state->task_slots = push_array(arena, W32_SMSV_TaskSlot, w32_smsv_state->task_slots_count);
+    w32_smsv_state->task_stripes = stripe_array_alloc(arena);
+  }
+  
+  // rjf: set up rings
+  w32_smsv_state->http_response_ring = guarded_ring_alloc(arena, KB(256));
+  w32_smsv_state->new_task_ring = guarded_ring_alloc(arena, KB(8));
+  w32_smsv_state->retry_arena = arena_alloc();
+}
+
+internal void
+smsv_async_tick(void)
+{
+  Temp scratch = scratch_begin(0, 0);
+  
+  //- rjf: pop new tasks from new task ring, gather them
+  W32_SMSV_TaskNode *first_request_task = 0;
+  W32_SMSV_TaskNode *last_request_task = 0;
+  U64 request_task_count = 0;
+  if(lane_idx() == 0)
+  {
+    for(B32 done = 0;!done;)
+    {
+      // rjf: get the next task ID to start
+      U64 task_id = 0;
+      {
+        RingGuard g = guarded_ring_open(w32_smsv_state->new_task_ring);
+        if(!guarded_ring_try_read_struct(&g, &task_id))
+        {
+          done = 1;
+        }
+        guarded_ring_close(&g);
+      }
+      
+      // rjf: gather
+      if(!done)
+      {
+        W32_SMSV_Task *task = (W32_SMSV_Task *)task_id;
+        W32_SMSV_TaskNode *n = push_array(scratch.arena, W32_SMSV_TaskNode, 1);
+        n->task = task;
+        SLLQueuePush(first_request_task, last_request_task, n);
+        request_task_count += 1;
+      }
+    }
+  }
+  lane_sync();
+  
+  //- rjf: gather tasks from the retry batch
+  if(lane_idx() == 0)
+  {
+    for(W32_SMSV_TaskNode *t = w32_smsv_state->first_retry_request; t != 0; t = t->next)
+    {
+      W32_SMSV_TaskNode *n = push_array(scratch.arena, W32_SMSV_TaskNode, 1);
+      n->task = t->task;
+      SLLQueuePush(first_request_task, last_request_task, n);
+      request_task_count += 1;
+    }
+    arena_clear(w32_smsv_state->retry_arena);
+    w32_smsv_state->first_retry_request = w32_smsv_state->last_retry_request = 0;
+  }
+  lane_sync();
+  
+  //- rjf: pop responses -> for any responses that suggest that a particular debug info
+  // is not present on the server, we need to generate new request nodes for those tasks,
+  // to fall back to subsequent server URLS.
+  W32_SMSV_TaskNode *first_finish_task = 0;
+  W32_SMSV_TaskNode *last_finish_task = 0;
+  U64 finish_task_count = 0;
+  if(lane_idx() == 0)
+  {
+    for(HTTP_Response r = {0}; http_pop_response(scratch.arena, w32_smsv_state->http_response_ring, &r, 0);)
+    {
+      W32_SMSV_Task *task = (W32_SMSV_Task *)r.id;
+      if(task != 0)
+      {
+        if(r.body.size != 0)
+        {
+          str8_list_push(task->arena, &task->file_pieces, str8_copy(task->arena, r.body));
+        }
+        if(r.has_more == 0 && task->status != W32_SMSV_TaskStatus_DoneDownloading)
+        {
+          task->status = W32_SMSV_TaskStatus_DoneDownloading;
+          W32_SMSV_TaskNode *n = push_array(scratch.arena, W32_SMSV_TaskNode, 1);
+          n->task = task;
+          SLLQueuePush(first_finish_task, last_finish_task, n);
+          finish_task_count += 1;
+        }
+      }
+    }
+  }
+  lane_sync();
+  
+  //- rjf: flatten all request tasks
+  W32_SMSV_Task **request_tasks = 0;
+  if(lane_idx() == 0)
+  {
+    request_tasks = push_array(scratch.arena, W32_SMSV_Task *, request_task_count);
+    {
+      U64 idx = 0;
+      for EachNode(n, W32_SMSV_TaskNode, first_request_task)
+      {
+        request_tasks[idx] = n->task;
+        idx += 1;
+      }
+    }
+  }
+  lane_sync_u64(&request_tasks, 0);
+  
+  //- rjf: do all request tasks
+  {
+    Rng1U64 range = lane_range(request_task_count);
+    for EachInRange(idx, range)
+    {
+      W32_SMSV_Task *task = request_tasks[idx];
+      
+      // rjf: adjust the server URL node that we're using for this task
+      if(task->status == W32_SMSV_TaskStatus_Failed || task->status == W32_SMSV_TaskStatus_Null)
+      {
+        if(task->current_server_url_node == 0)
+        {
+          task->current_server_url_node = w32_smsv_state->symbol_server_urls.first;
+        }
+        else
+        {
+          task->current_server_url_node = task->current_server_url_node->next;
+        }
+      }
+      
+      // rjf: get server URL
+      String8 server_url = {0};
+      if(task->current_server_url_node != 0)
+      {
+        server_url = task->current_server_url_node->string;
+      }
+      
+      // rjf: kick off request if we have a server URL
+      if(server_url.size != 0)
+      {
+        task->status = W32_SMSV_TaskStatus_Requested;
+        Guid guid = task->guid;
+        U64 age = task->age;
+        String8 debug_info_unique_identifier = {0};
+        {
+          debug_info_unique_identifier = str8f(scratch.arena, "%08X%04X%04X%02X%02X%02X%02X%02X%02X%02X%02X%I64x",
+                                               guid.data1, guid.data2, guid.data3, guid.data4[0], guid.data4[1], guid.data4[2], guid.data4[3], guid.data4[4], guid.data4[5], guid.data4[6], guid.data4[7], age);
+        }
+        HTTP_RequestParams params =
+        {
+          .id = (U64)task,
+          .method = HTTP_Method_Get,
+          .url = str8f(scratch.arena, "%S/%S/%S/%S", server_url, task->dbg_name, debug_info_unique_identifier, task->dbg_name),
+        };
+        if(!http_push_request(w32_smsv_state->http_response_ring, &params, 0))
+        {
+          W32_SMSV_TaskNode *n = push_array(w32_smsv_state->retry_arena, W32_SMSV_TaskNode, 1);
+          SLLQueuePush(w32_smsv_state->first_retry_request, w32_smsv_state->last_retry_request, n);
+          n->task = task;
+          ins_atomic_u32_eval_assign(&async_loop_again, 1);
+          cond_var_broadcast(async_tick_start_cond_var);
+        }
+      }
+      
+      // rjf: if we don't have a server URL, mark this task as failed
+      else
+      {
+        task->status = W32_SMSV_TaskStatus_Failed;
+        W32_SMSV_TaskNode *n = push_array(scratch.arena, W32_SMSV_TaskNode, 1);
+        n->task = task;
+        SLLQueuePush(first_finish_task, last_finish_task, n);
+        finish_task_count += 1;
+      }
+    }
+  }
+  
+  //- rjf: flatten all finish tasks
+  W32_SMSV_Task **finish_tasks = 0;
+  if(lane_idx() == 0)
+  {
+    finish_tasks = push_array(scratch.arena, W32_SMSV_Task *, finish_task_count);
+    {
+      U64 idx = 0;
+      for EachNode(n, W32_SMSV_TaskNode, first_finish_task)
+      {
+        finish_tasks[idx] = n->task;
+        idx += 1;
+      }
+    }
+  }
+  lane_sync_u64(&finish_tasks, 0);
+  
+  //- rjf: do all finish tasks
+  {
+    Rng1U64 range = lane_range(finish_task_count);
+    for EachInRange(idx, range)
+    {
+      W32_SMSV_Task *task = finish_tasks[idx];
+      
+      // rjf: successful task -> write to local path
+      if(task->status != W32_SMSV_TaskStatus_Failed)
+      {
+        String8 directory = str8_chop_last_slash(task->local_path);
+        for(U64 slash_pos = 0; slash_pos <= directory.size;)
+        {
+          slash_pos = str8_find_needle(directory, slash_pos+1, s("/"), StringMatchFlag_SlashInsensitive);
+          String8 root_dir = str8_substr(directory, r1u64(0, slash_pos));
+          make_directory(root_dir);
+          if(slash_pos == directory.size)
+          {
+            break;
+          }
+        }
+        write_data_list_to_file_path(task->local_path, task->file_pieces);
+      }
+      
+      // rjf: evict task from cache
+      U64 hash = u64_hash_from_str8(task->local_path);
+      U64 slot_idx = hash%w32_smsv_state->task_slots_count;
+      W32_SMSV_TaskSlot *slot = &w32_smsv_state->task_slots[slot_idx];
+      Stripe *stripe = stripe_from_slot_idx(&w32_smsv_state->task_stripes, slot_idx);
+      RWMutexScope(stripe->rw_mutex, 1)
+      {
+        DLLRemove(slot->first, slot->last, task);
+        arena_release(task->arena);
+      }
+    }
+  }
+  
+  scratch_end(scratch);
 }
 
 internal String8
-smsv_local_path_from_key(Arena *arena, String8 module_name, String8 dbg_name, Guid guid, U64 age)
+smsv_local_path_from_key(Arena *arena, String8 dbg_name, Guid guid, U64 age)
 {
   String8 result = {0};
   Temp scratch = scratch_begin(&arena, 1);
@@ -96,19 +331,103 @@ smsv_local_path_from_key(Arena *arena, String8 module_name, String8 dbg_name, Gu
     // rjf: form unique identifier based on guid
     String8 debug_info_unique_identifier = {0};
     {
-      debug_info_unique_identifier = str8f(scratch.arena, "%08X%04X%04X%02X%02X%02X%02X%02X%02X%02X%02X%x",
+      debug_info_unique_identifier = str8f(scratch.arena, "%08X%04X%04X%02X%02X%02X%02X%02X%02X%02X%02X%I64x",
                                            guid.data1, guid.data2, guid.data3, guid.data4[0], guid.data4[1], guid.data4[2], guid.data4[3], guid.data4[4], guid.data4[5], guid.data4[6], guid.data4[7], age);
     }
     
     // rjf: form full path
     if(debug_info_unique_identifier.size != 0)
     {
-      // TODO(rjf): assuming 'stripped' for now
       String8 symbol_cache_path = w32_smsv_state->symbol_cache_path;
-      String8 path = str8f(scratch.arena, "%S/%S/%S/stripped/%S", symbol_cache_path, module_name, debug_info_unique_identifier, dbg_name);
+      String8 path = str8f(scratch.arena, "%S/%S/%S/%S", symbol_cache_path, dbg_name, debug_info_unique_identifier, dbg_name);
       result = path_normalized_from_string(arena, path);
     }
+    
+    // rjf: determine if we already have this debug info stored locally
+    B32 already_cached_locally = (properties_from_file_path(result).modified != 0);
+    
+    // rjf: if not cached: record (local path -> download task) mapping
+    B32 task_is_new = 0;
+    U64 task_id = 0;
+    if(!already_cached_locally)
+    {
+      U64 hash = u64_hash_from_str8(result);
+      U64 slot_idx = hash%w32_smsv_state->task_slots_count;
+      W32_SMSV_TaskSlot *slot = &w32_smsv_state->task_slots[slot_idx];
+      Stripe *stripe = stripe_from_slot_idx(&w32_smsv_state->task_stripes, slot_idx);
+      for(B32 write_mode = 0; write_mode <= 1; write_mode += 1)
+      {
+        B32 already_exists = 0;
+        RWMutexScope(stripe->rw_mutex, write_mode)
+        {
+          W32_SMSV_Task *node = 0;
+          for(W32_SMSV_Task *n = slot->first; n != 0; n = n->next)
+          {
+            if(MemoryMatchStruct(&n->guid, &guid))
+            {
+              already_exists = 1;
+              node = n;
+              break;
+            }
+          }
+          if(node == 0 && write_mode)
+          {
+            Arena *arena = arena_alloc();
+            node = push_array(arena, W32_SMSV_Task, 1);
+            node->arena = arena;
+            node->local_path = str8_copy(arena, result);
+            node->dbg_name = str8_copy(arena, dbg_name);
+            node->guid = guid;
+            node->age = age;
+            DLLPushBack(slot->first, slot->last, node);
+            task_is_new = 1;
+            task_id = (U64)node;
+          }
+        }
+        if(already_exists)
+        {
+          break;
+        }
+      }
+    }
+    
+    // rjf: if the task is new -> push request to start task
+    if(task_is_new)
+    {
+      RingGuard g = guarded_ring_open(w32_smsv_state->new_task_ring);
+      guarded_ring_write_struct_or_wait(&g, &task_id, max_U64);
+      guarded_ring_close(&g);
+      ins_atomic_u32_eval_assign(&async_loop_again, 1);
+      cond_var_broadcast(async_tick_start_cond_var);
+    }
   }
   scratch_end(scratch);
   return result;
 }
+
+internal SMSV_Status
+smsv_status_from_local_path(String8 path)
+{
+  SMSV_Status status = SMSV_Status_Null;
+  {
+    Temp scratch = scratch_begin(0, 0);
+    String8 path_normalized = path_normalized_from_string(scratch.arena, path);
+    U64 hash = u64_hash_from_str8(path_normalized);
+    U64 slot_idx = hash%w32_smsv_state->task_slots_count;
+    W32_SMSV_TaskSlot *slot = &w32_smsv_state->task_slots[slot_idx];
+    Stripe *stripe = stripe_from_slot_idx(&w32_smsv_state->task_stripes, slot_idx);
+    RWMutexScope(stripe->rw_mutex, 0)
+    {
+      for(W32_SMSV_Task *n = slot->first; n != 0; n = n->next)
+      {
+        if(str8_match(n->local_path, path, 0))
+        {
+          status = SMSV_Status_Pending;
+          break;
+        }
+      }
+    }
+    scratch_end(scratch);
+  }
+  return status;
+}

src/win32/symbol_server/win32_symbol_server.h

@@ -4,12 +4,62 @@
 #ifndef WIN32_SYMBOL_SERVER_H
 #define WIN32_SYMBOL_SERVER_H
 
+typedef enum W32_SMSV_TaskStatus
+{
+  W32_SMSV_TaskStatus_Null,
+  W32_SMSV_TaskStatus_Requested,
+  W32_SMSV_TaskStatus_DoneDownloading,
+  W32_SMSV_TaskStatus_Failed,
+}
+W32_SMSV_TaskStatus;
+
+typedef struct W32_SMSV_Task W32_SMSV_Task;
+struct W32_SMSV_Task
+{
+  W32_SMSV_Task *next;
+  W32_SMSV_Task *prev;
+  
+  // rjf: key / parameters
+  Arena *arena;
+  String8 local_path;
+  String8 dbg_name;
+  Guid guid;
+  U64 age;
+  
+  // rjf: working state
+  W32_SMSV_TaskStatus status;
+  String8Node *current_server_url_node;
+  String8List file_pieces;
+};
+
+typedef struct W32_SMSV_TaskNode W32_SMSV_TaskNode;
+struct W32_SMSV_TaskNode
+{
+  W32_SMSV_TaskNode *next;
+  W32_SMSV_Task *task;
+};
+
+typedef struct W32_SMSV_TaskSlot W32_SMSV_TaskSlot;
+struct W32_SMSV_TaskSlot
+{
+  W32_SMSV_Task *first;
+  W32_SMSV_Task *last;
+};
+
 typedef struct W32_SMSV_State W32_SMSV_State;
 struct W32_SMSV_State
 {
   Arena *arena;
   String8 symbol_cache_path;
   String8List symbol_server_urls;
+  U64 task_slots_count;
+  W32_SMSV_TaskSlot *task_slots;
+  StripeArray task_stripes;
+  GuardedRing *http_response_ring;
+  GuardedRing *new_task_ring;
+  Arena *retry_arena;
+  W32_SMSV_TaskNode *first_retry_request;
+  W32_SMSV_TaskNode *last_retry_request;
 };
 
 global W32_SMSV_State *w32_smsv_state = 0;