ed/raddebugger
1
0
Fork 0
mirror of https://github.com/Ed94/raddebugger.git synced 2026-08-14 09:18:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

checkpoint on symbol server

Browse Source
This commit is contained in:
Ryan Fleury
2026-06-08 17:32:07 -07:00
parent 536ca53d18
commit b22f9f2d8b
21 changed files with 736 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
project.raddbg
+6
View File
@@ -47,3 +47,9 @@ target:
working_directory: build
arguments: "--bin --quiet --rdi --out:C:/devel/raddebugger_stable/build/radbin.rdi --thread_count:1 C:/devel/raddebugger_stable/build/radbin.exe"
}
target:
{
executable: "build/raddbg.exe"
working_directory: "build/"
arguments: "--bin --rdi C:\\SymbolCache\\ntdll.pdb\\180BF1B90AA75697D0EFEA5E5630AC7E1\\ntdll.pdb --thread_count=1"
}
src/base/base_entry_point.c
+3
View File
@@ -224,6 +224,9 @@ async_thread_entry_point(void *params)
#if defined(HTTP_H)
http_async_tick();
#endif
#if defined(SYMBOL_SERVER_H)
smsv_async_tick();
#endif
#if defined(DBG_INFO_H)
di_async_tick();
#endif
src/base/base_thread_context.c
-1
View File
@@ -79,7 +79,6 @@ tctx_get_scratch(Arena **conflicts, U64 count)
break;
}
}
Assert(result);
return result;
}
src/dbg_engine/dbg_engine_ctrl.c
+3 -1
View File
@@ -2685,10 +2685,12 @@ d_ctrl_thread__module_open(D_Handle process, D_Handle module, U64 base_vaddr, DM
//////////////////////////////
//- rjf: no found debug info path -> try to fall back on symbol server
//
#if 0
if(initial_debug_info_path.size == 0)
{
initial_debug_info_path = smsv_local_path_from_key(arena, str8_skip_last_slash(module_info->module_path), str8_skip_last_slash(module_info->debug_info_path), module_info->debug_info_guid, module_info->debug_info_age);
initial_debug_info_path = smsv_local_path_from_key(arena, str8_skip_last_slash(module_info->debug_info_path), module_info->debug_info_guid, module_info->debug_info_age);
}
#endif
//////////////////////////////
//- rjf: write 1 at attachment marker, to signify attachment
src/dbg_info/dbg_info.c
+19 -5
View File
@@ -680,8 +680,22 @@ di_async_tick(void)
}
}
//- rjf: check with the symbol server to see if we are actively downloading O.G.
B32 og_is_downloading = 0;
{
B32 file_is_present = (properties_from_file_path(og_path).modified != 0);
if(!file_is_present)
{
SMSV_Status status = smsv_status_from_local_path(og_path);
if(status == SMSV_Status_Pending)
{
og_is_downloading = 1;
}
}
}
//- rjf: analyze O.G. debug info
if(!t->og_analyzed)
if(!t->og_analyzed && !og_is_downloading)
{
t->og_analyzed = 1;
File file = file_open(AccessFlag_ShareRead|AccessFlag_Read, og_path);
@@ -714,7 +728,7 @@ di_async_tick(void)
}
//- rjf: determine if RDI is stale
if(!t->rdi_analyzed)
if(!t->rdi_analyzed && !og_is_downloading)
{
t->rdi_analyzed = 1;
File file = file_open(AccessFlag_ShareRead|AccessFlag_Read, rdi_path);
@@ -737,7 +751,7 @@ di_async_tick(void)
B32 rdi_is_stale = t->rdi_is_stale;
//- rjf: calculate thread counts for conversion processes
if(!og_is_rdi && rdi_is_stale && t->thread_count == 0)
if(!og_is_rdi && rdi_is_stale && t->thread_count == 0 && !og_is_downloading)
{
U64 thread_count = 1;
U64 max_thread_count = get_system_info()->logical_processor_count/2;
@@ -768,7 +782,7 @@ di_async_tick(void)
//- rjf: if this conversion will overwrite an RDI we already have in cache,
// then we need to evict the old one from the cache.
B32 ready_to_launch_conversion = (threads_available && !og_is_rdi && rdi_is_stale && t->thread_count != 0 && t->status != DI_LoadTaskStatus_Active);
B32 ready_to_launch_conversion = (!og_is_downloading && threads_available && !og_is_rdi && rdi_is_stale && t->thread_count != 0 && t->status != DI_LoadTaskStatus_Active);
if(ready_to_launch_conversion)
{
U64 path2key_hash = u64_hash_from_str8(og_path);
@@ -863,7 +877,7 @@ di_async_tick(void)
//- rjf: if the RDI for this task is not stale, then we're already done - mark this
// task as done & prepped for storing into the cache
if(!rdi_is_stale)
if(!rdi_is_stale && !og_is_downloading)
{
t->status = DI_LoadTaskStatus_Done;
}
src/dbg_info/dbg_info.h
+1
View File
@@ -140,6 +140,7 @@ struct DI_LoadTask
B32 og_analyzed;
B32 og_is_rdi;
U64 og_size;
B32 og_is_downloading;
B32 rdi_analyzed;
B32 rdi_is_stale;
src/http/http.c
+18
View File
@@ -31,3 +31,21 @@ http_status_kind_from_code(HTTP_StatusCode code)
}
return kind;
}
////////////////////////////////////////////////////////////////
//~ rjf: Synchronous Request Helper
internal HTTP_Response
http_request(Arena *arena, HTTP_RequestParams *params)
{
HTTP_Response response = {0};
Temp scratch = scratch_begin(&arena, 1);
GuardedRing *gr = guarded_ring_alloc(scratch.arena, KB(64));
{
http_push_request(gr, params, max_U64);
http_pop_response(arena, gr, &response, max_U64);
}
guarded_ring_release(gr);
scratch_end(scratch);
return response;
}
src/http/http.h
+6
View File
@@ -146,6 +146,7 @@ typedef struct HTTP_Response HTTP_Response;
struct HTTP_Response
{
U64 id;
B32 has_more;
HTTP_StatusCode code;
String8 body;
};
@@ -155,6 +156,11 @@ struct HTTP_Response
internal HTTP_StatusKind http_status_kind_from_code(HTTP_StatusCode code);
////////////////////////////////////////////////////////////////
//~ rjf: Synchronous Request Helper
internal HTTP_Response http_request(Arena *arena, HTTP_RequestParams *params);
////////////////////////////////////////////////////////////////
//~ rjf: @per_os_impl Top-Level Layer Calls
src/pdb/pdb_parse.c
+39 -5
View File
@@ -5,17 +5,21 @@
//~ PDB Parser Functions
internal PDB_Info*
pdb_info_from_data(Arena *arena, String8 data){
pdb_info_from_data(Arena *arena, String8 data)
{
ProfBegin("pdb_info_from_data");
PDB_Info *result = push_array(arena, PDB_Info, 1);
// get header
PDB_InfoHeader *header = 0;
if (data.size >= sizeof(*header)){
if(data.size >= sizeof(*header))
{
header = (PDB_InfoHeader *)data.str;
}
PDB_Info *result = 0;
if (header != 0){
// rjf: parse info given header
if(header != 0)
{
// read guid
Guid *auth_guid = 0;
U32 after_auth_guid_off = sizeof(*header);
@@ -122,7 +126,6 @@ pdb_info_from_data(Arena *arena, String8 data){
}
ProfEnd();
return(result);
}
@@ -757,6 +760,37 @@ pdb_comp_unit_contribution_array_from_data(Arena *arena, String8 data, COFF_Sect
return result;
}
internal PDB_CompUnitContribution *
pdb_comp_unit_contribution_from_voff__binary_search(PDB_CompUnitContributionArray *array, U64 voff)
{
PDB_CompUnitContribution *result = 0;
if(array->count != 0)
{
U64 first_idx = 0;
U64 last_idx = array->count-1;
for(;first_idx < last_idx;)
{
U64 mid_idx = (last_idx + first_idx) / 2;
U64 mid_voff_first = array->contributions[mid_idx].voff_first;
U64 mid_voff_opl = array->contributions[mid_idx].voff_opl;
if(voff < mid_voff_first)
{
last_idx = mid_idx;
}
else if(mid_voff_opl <= voff)
{
first_idx = mid_idx+1;
}
else
{
result = &array->contributions[mid_idx];
break;
}
}
}
return result;
}
////////////////////////////////
//~ PDB Dbi Functions
src/pdb/pdb_parse.h
+1
View File
@@ -212,6 +212,7 @@ internal PDB_CompUnitArray* pdb_comp_unit_array_from_data(Arena *arena,
String8 module_info_data);
internal PDB_CompUnitContributionArray pdb_comp_unit_contribution_array_from_data(Arena *arena, String8 seccontrib_data, COFF_SectionHeaderArray sections);
internal PDB_CompUnitContribution *pdb_comp_unit_contribution_from_voff__binary_search(PDB_CompUnitContributionArray *array, U64 voff);
////////////////////////////////
//~ PDB Dbi Functions
src/raddbg/raddbg_core.c
+27 -7
View File
@@ -10684,7 +10684,8 @@ rd_init(CmdLine *cmdln)
// rjf: set up update check http ring & send request
{
rd_state->update_check_http_ring = guarded_ring_alloc(arena, KB(8));
rd_state->update_check_arena = arena_alloc();
rd_state->update_check_http_ring = guarded_ring_alloc(rd_state->update_check_arena, KB(8));
HTTP_RequestParams p =
{
.id = 1,
@@ -11102,12 +11103,24 @@ rd_frame(void)
//
if(!rd_state->got_update_check)
{
Temp scratch = scratch_begin(0, 0);
HTTP_Response response = {0};
if(http_pop_response(scratch.arena, rd_state->update_check_http_ring, &response, 0))
// rjf: pop all responses that we can
B32 done = 0;
for(HTTP_Response response = {0}; http_pop_response(rd_state->update_check_arena, rd_state->update_check_http_ring, &response, 0);)
{
str8_list_push(rd_state->update_check_arena, &rd_state->update_check_response_body_pieces, response.body);
if(response.has_more == 0)
{
done = 1;
}
}
// rjf: if we're done -> parse response & determine if there is a new version
if(done)
{
rd_state->got_update_check = 1;
MD_Node *root = md_tree_from_string(scratch.arena, response.body)->first;
Temp scratch = scratch_begin(0, 0);
String8 response_body = str8_list_join(rd_state->update_check_arena, &rd_state->update_check_response_body_pieces, 0);
MD_Node *root = md_tree_from_string(scratch.arena, response_body)->first;
MD_Node *html_url = md_child_from_string(root, s("html_url"), 0);
String8 newest_release_url = html_url->first->string;
U64 tag_pos = str8_find_needle(newest_release_url, 0, s("tag/v"), 0);
@@ -11120,9 +11133,14 @@ rd_frame(void)
{
rd_state->newer_update_available = 1;
}
}
guarded_ring_release(rd_state->update_check_http_ring);
arena_release(rd_state->update_check_arena);
rd_state->update_check_arena = 0;
rd_state->update_check_http_ring = 0;
MemoryZeroStruct(&rd_state->update_check_response_body_pieces);
scratch_end(scratch);
}
}
//////////////////////////////
//- rjf: iterate all tabs, touch their view-states
@@ -17380,7 +17398,9 @@ rd_frame(void)
D_Entity *module = d_entity_from_handle(evt->entity);
D_Entity *debug_info_path = d_entity_child_from_kind(module, D_EntityKind_DebugInfoPath);
String8 new_path = debug_info_path->string;
if(new_path.size != 0 && file_path_exists(new_path))
B32 file_exists = file_path_exists(new_path);
B32 file_is_pending = (!file_exists ? smsv_status_from_local_path(new_path) == SMSV_Status_Pending : 0);
if(new_path.size != 0 && (file_exists || file_is_pending))
{
CFG_NodePtrList dbg_infos = cfg_node_top_level_list_from_string(scratch.arena, str8_lit("debug_info"));
B32 path_found = 0;
src/raddbg/raddbg_core.h
+2
View File
@@ -397,6 +397,8 @@ struct RD_State
// rjf: update checker
B32 got_update_check;
B32 newer_update_available;
Arena *update_check_arena;
String8List update_check_response_body_pieces;
GuardedRing *update_check_http_ring;
// rjf: config bucket paths
src/raddbg/raddbg_main.c
+29
View File
@@ -5,6 +5,7 @@
//~ rjf: post-0.9.26 TODO notes
//
// [ ] unwinding crash - f5 on bp in w32_http_status_callback to repro
// [ ] unwinding in jit page - fail
// [ ] jeff stepping bug
// [ ] wassim memory bug
// [ ] switch off spoofs for step-over
@@ -514,6 +515,34 @@ entry_point(CmdLine *cmd_line)
{
Temp scratch = scratch_begin(0, 0);
//- TODO(rjf): test symbol server download
#if 0
{
GuardedRing *out_ring = guarded_ring_alloc(scratch.arena, KB(64));
{
HTTP_RequestParams p =
{
.id = 1,
.method = HTTP_Method_Get,
.url = s("https://msdl.microsoft.com/download/symbols/ntdll.pdb/E84AF4A1E3CDFABC99A41209581EB6861/ntdll.pdb"),
};
http_push_request(out_ring, &p, max_U64);
String8List pieces = {0};
for(HTTP_Response r = {0}; http_pop_response(scratch.arena, out_ring, &r, max_U64);)
{
str8_list_push(scratch.arena, &pieces, r.body);
if(r.has_more == 0)
{
int x = 0;
break;
}
}
int x = 0;
}
guarded_ring_release(out_ring);
}
#endif
//- rjf: unpack command line arguments
ExecMode exec_mode = ExecMode_Normal;
B32 auto_run = 0;
src/rdi_from_pdb/rdi_from_pdb.c
+84 -13
View File
@@ -397,22 +397,18 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
{
Temp scratch = scratch_begin(&arena, 1);
Arena *scratch_conflicts[2] = { scratch.arena, arena };
//////////////////////////////////////////////////////////////
//- rjf: do base MSF parse
//
MSF_Parsed *msf = 0;
ProfScope("do base MSF parse")
{
Temp scratch2 = scratch_begin(scratch_conflicts, ArrayCount(scratch_conflicts));
// rjf: setup output buckets
MSF_RawStreamTable *msf_raw_stream_table = 0;
U64 *msf_stream_take_counter = 0;
if(lane_idx() == 0)
{
msf_raw_stream_table = msf_raw_stream_table_from_data(scratch2.arena, params->input_pdb_data);
msf_raw_stream_table = msf_raw_stream_table_from_data(scratch.arena, params->input_pdb_data);
msf = push_array(scratch.arena, MSF_Parsed, 1);
msf->page_size = msf_raw_stream_table->page_size;
msf->page_count = msf_raw_stream_table->total_page_count;
@@ -437,8 +433,6 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
}
}
lane_sync();
scratch_end(scratch2);
}
lane_sync();
@@ -677,7 +671,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
//
U64 exe_voff_max = 0;
{
Temp scratch2 = scratch_begin(scratch_conflicts, ArrayCount(scratch_conflicts));
Temp scratch2 = scratch_begin(&scratch.arena, 1);
// rjf: set up
U64 *lane_voff_maxes = 0;
@@ -827,7 +821,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
//- rjf: do wide gather
ProfScope("do wide gather")
{
Temp scratch2 = scratch_begin(scratch_conflicts, ArrayCount(scratch_conflicts));
Temp scratch2 = scratch_begin(&scratch.arena, 1);
//- rjf: build local hash table to dedup files within this lane
U64 hit_path_slots_count = 4096;
@@ -1758,7 +1752,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
}
//- rjf: walk dependent types, push to chain
Temp scratch2 = scratch_begin(scratch_conflicts, ArrayCount(scratch_conflicts));
Temp scratch2 = scratch_begin(&scratch.arena, 1);
P2R_TypeIdChain start_walk_task = {0, itype};
P2R_TypeIdChain *first_walk_task = &start_walk_task;
P2R_TypeIdChain *last_walk_task = &start_walk_task;
@@ -1767,7 +1761,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
walk_task = walk_task->next)
{
CV_TypeId walk_itype = itype_fwd_map[walk_task->itype] ? itype_fwd_map[walk_task->itype] : walk_task->itype;
if(walk_itype < tpi_leaf->itype_first)
if(walk_itype < tpi_leaf->itype_first || tpi_leaf->itype_opl <= walk_itype)
{
continue;
}
@@ -2041,7 +2035,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
U64 all_namespace_slots_count = 0;
ProfScope("gather all unique namespaces from types")
{
Temp scratch2 = scratch_begin(scratch_conflicts, ArrayCount(scratch_conflicts));
Temp scratch2 = scratch_begin(&scratch.arena, 1);
//- rjf: find all unique namespaces on this lane
U64 namespace_slots_count = (U64)itype_opl / lane_count();
@@ -2874,7 +2868,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
RDIM_SubsetFlag_Types))
ProfScope("gather all namespaces which are only encoded in symbols (not found in types)")
{
Temp scratch2 = scratch_begin(scratch_conflicts, ArrayCount(scratch_conflicts));
Temp scratch2 = scratch_begin(&scratch.arena, 1);
U64 lane_namespace_slots_count = 4096;
String8Node **lane_namespace_slots = push_array(scratch2.arena, String8Node *, lane_namespace_slots_count);
@@ -3537,6 +3531,7 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
if(n->voff == voff)
{
link_name = n->name;
ins_atomic_u64_inc_eval(&n->referencing_symbol_count);
break;
}
}
@@ -4183,6 +4178,82 @@ p2r_convert(Arena *arena, P2R_ConvertParams *params)
}
lane_sync();
//////////////////////////////////////////////////////////////
//- rjf: produce stub procedures from all pub32 symbols
//
if(lane_idx() == 0)
{
CV_SymParsed *sym = all_syms[0];
for(CV_RecIter iter = {0}; cv_rec_next(sym->data, &sym->sym_ranges, 0, &iter);)
{
RDIM_Unit *sym_unit = &all_units_ptr->first->v[0];
RDIM_SymbolChunkList *sym_procedures = &sym_unit->procedures;
RDIM_SymbolChunkList *sym_global_variables = &sym_unit->global_variables;
RDIM_ScopeChunkList *sym_scopes = &sym_unit->scopes;
switch(iter.kind)
{
default:{}break;
case CV_SymKind_PUB32:
{
// rjf: unpack record
CV_SymPub32 *pub32 = (CV_SymPub32 *)iter.struct_base;
String8 name = str8_cstring_capped(pub32+1, iter.opl);
COFF_SectionHeader *section = (0 < pub32->sec && pub32->sec <= coff_sections.count) ? &coff_sections.v[pub32->sec-1] : 0;
U64 voff = 0;
if(section != 0)
{
voff = section->voff + pub32->off;
}
// rjf: determine if this pub32 was covered by another symbol record
B32 was_in_other_record = 0;
{
U64 hash = p2r_hash_from_voff(voff);
U64 slot_idx = hash%link_name_map->buckets_count;
for(P2R_LinkNameNode *n = link_name_map->buckets[slot_idx]; n != 0; n = n->next)
{
if(n->voff == voff)
{
was_in_other_record = (n->referencing_symbol_count != 0);
break;
}
}
}
// rjf: find compilation unit contribution to determine pub32's size
U64 voff_opl = voff+1;
if(!was_in_other_record)
{
PDB_CompUnitContribution *contrib = pdb_comp_unit_contribution_from_voff__binary_search(comp_unit_contributions, voff);
if(contrib != 0)
{
voff_opl = contrib->voff_opl;
}
}
// rjf: if this pub32 was *not* covered by another symbol record, build a symbol for it
if(!was_in_other_record)
{
B32 is_procedure = (pub32->flags & (CV_Pub32Flag_Code|CV_Pub32Flag_Function));
RDIM_SymbolChunkList *symbols = is_procedure ? sym_procedures : sym_global_variables;
RDIM_Symbol *symbol = rdim_symbol_chunk_list_push(arena, symbols, 64);
symbol->name = name;
symbol->link_name = name;
if(is_procedure)
{
RDIM_Scope *scope = rdim_scope_chunk_list_push(arena, sym_scopes, 64);
scope->symbol = symbol;
RDIM_Rng1U64 range = {voff, voff_opl};
rdim_scope_push_voff_range(arena, sym_scopes, scope, range);
symbol->root_scope = scope;
}
}
}break;
}
}
}
lane_sync();
//////////////////////////////////////////////////////////////
//- rjf: upgrade namespace nodes with scopes info, if they match a scope
//
src/rdi_from_pdb/rdi_from_pdb.h
+1
View File
@@ -29,6 +29,7 @@ struct P2R_LinkNameNode
P2R_LinkNameNode *next;
U64 voff;
String8 name;
U64 referencing_symbol_count;
};
typedef struct P2R_LinkNameMap P2R_LinkNameMap;
src/symbol_server/symbol_server.h
+10 -1
View File
@@ -4,7 +4,16 @@
#ifndef SYMBOL_SERVER_H
#define SYMBOL_SERVER_H
typedef enum SMSV_Status
{
SMSV_Status_Null,
SMSV_Status_Pending,
}
SMSV_Status;
internal void smsv_init(void);
internal String8 smsv_local_path_from_key(Arena *arena, String8 module_name, String8 dbg_name, Guid guid, U64 age);
internal void smsv_async_tick(void);
internal String8 smsv_local_path_from_key(Arena *arena, String8 dbg_name, Guid guid, U64 age);
internal SMSV_Status smsv_status_from_local_path(String8 path);
#endif // SYMBOL_SERVER_H
src/symbol_server/symbol_server_stub.c
+13 -1
View File
@@ -6,8 +6,20 @@ smsv_init(void)
{
}
internal void
smsv_async_tick(void)
{
}
internal String8
smsv_local_path_from_key(Arena *arena, String8 module_name, String8 dbg_name, Guid guid, U64 age)
smsv_local_path_from_key(Arena *arena, String8 dbg_name, Guid guid, U64 age)
{
return s("");
}
internal SMSV_Status
smsv_status_from_local_path(String8 path)
{
SMSV_Status status = SMSV_Status_Null;
return status;
}
src/win32/http/win32_http.c
+95 -57
View File
@@ -293,6 +293,7 @@ http_async_tick(void)
}
req->hConnect = hConnect;
req->hRequest = hRequest;
req->arena_start_body_read_pos = arena_pos(req->arena);
//- rjf: send request
if(good)
@@ -324,6 +325,95 @@ http_async_tick(void)
}
}
//- rjf: push response output
{
Rng1U64 range = lane_range(ArrayCount(w32_http_state->active_requests));
for EachInRange(idx, range)
{
W32_HTTP_Request *req = &w32_http_state->active_requests[idx];
W32_HTTP_RequestStatus status = ins_atomic_u32_eval(&req->status);
if(req->active && (status == W32_HTTP_RequestStatus_Done ||
status == W32_HTTP_RequestStatus_Failed ||
req->finished_body_pieces.total_size >= req->out_ring->ring->size/2))
{
Temp scratch = scratch_begin(0, 0);
// rjf: figure out status code
if(req->status_code == 0)
{
DWORD status_code = 0;
DWORD status_code_size = sizeof(status_code);
WinHttpQueryHeaders(req->hRequest,
WINHTTP_QUERY_STATUS_CODE|WINHTTP_QUERY_FLAG_NUMBER,
WINHTTP_HEADER_NAME_BY_INDEX,
&status_code, &status_code_size, WINHTTP_NO_HEADER_INDEX);
req->status_code = status_code;
}
// rjf: join all currently read body contents
String8 body = str8_list_join(scratch.arena, &req->finished_body_pieces, 0);
// rjf: clear current body output
arena_pop_to(req->arena, req->arena_start_body_read_pos);
MemoryZeroStruct(&req->finished_body_pieces);
// rjf: compute size per record header
U64 max_record_size = req->out_ring->ring->size;
U64 bytes_per_record_header = sizeof(U64) + sizeof(U64) + sizeof(B32) + sizeof(HTTP_StatusCode);
// rjf: push response records
for(U64 off = 0, next_off = 0; off <= body.size; off = next_off)
{
// rjf: determine how much of the body we can fit into this record
U64 body_data_size = (body.size - off);
if(bytes_per_record_header + body_data_size > max_record_size)
{
body_data_size = max_record_size - bytes_per_record_header;
}
// rjf: advance by that amount of the body
next_off = off + body_data_size;
// rjf: serialize this record
String8 record_data = {0};
{
String8List pieces = {0};
U64 record_size = bytes_per_record_header + (next_off - off);
B32 is_last = (next_off == body.size && (status == W32_HTTP_RequestStatus_Done || status == W32_HTTP_RequestStatus_Failed));
B32 has_more = !is_last;
str8_list_push(scratch.arena, &pieces, str8_struct(&record_size));
str8_list_push(scratch.arena, &pieces, str8_struct(&req->id));
str8_list_push(scratch.arena, &pieces, str8_struct(&has_more));
str8_list_push(scratch.arena, &pieces, str8_struct(&req->status_code));
str8_list_push(scratch.arena, &pieces, str8_substr(body, r1u64(off, next_off)));
record_data = str8_list_join(scratch.arena, &pieces, 0);
}
// rjf: push this record
{
// TODO(rjf): can't wait here technically, because we might be waiting for other
// work in the async threads to pop! so we'll just deadlock right here. two options:
//
// (a) http requests need to be on another timeline
// (b) we need to prepare for this to fail & batch it up or something.
//
RingGuard g = guarded_ring_open(req->out_ring);
guarded_ring_write_string_or_wait(&g, record_data, max_U64);
guarded_ring_close(&g);
}
// rjf: cancel on no movement
if(next_off == off)
{
break;
}
}
scratch_end(scratch);
}
}
}
//- rjf: kick off body size queries
{
Rng1U64 range = lane_range(ArrayCount(w32_http_state->active_requests));
@@ -364,62 +454,10 @@ http_async_tick(void)
{
W32_HTTP_Request *req = &w32_http_state->active_requests[idx];
W32_HTTP_RequestStatus status = ins_atomic_u32_eval(&req->status);
if(req->active &&
if(req->active && req->finished_body_pieces.total_size == 0 && req->next_body_piece_size == 0 &&
(status == W32_HTTP_RequestStatus_Done ||
status == W32_HTTP_RequestStatus_Failed))
{
Temp scratch = scratch_begin(0, 0);
B32 good = (status == W32_HTTP_RequestStatus_Done);
//- rjf: unpack
HINTERNET hRequest = req->hRequest;
//- rjf: read response code
HTTP_StatusCode code = 0;
if(good)
{
DWORD status_code = 0;
DWORD status_code_size = sizeof(status_code);
WinHttpQueryHeaders(hRequest,
WINHTTP_QUERY_STATUS_CODE|WINHTTP_QUERY_FLAG_NUMBER,
WINHTTP_HEADER_NAME_BY_INDEX,
&status_code, &status_code_size, WINHTTP_NO_HEADER_INDEX);
code = status_code;
}
//- rjf: serialize response
String8 response_data = {0};
U64 response_data_min_size = sizeof(U64) + sizeof(U64) + sizeof(HTTP_StatusCode) + sizeof(U64);
U64 response_data_max_size = req->out_ring->ring->size;
B32 response_is_possible = (response_data_min_size < response_data_max_size);
if(response_is_possible)
{
U64 body_size = req->finished_body_pieces.total_size;
String8List pieces = {0};
str8_list_push(scratch.arena, &pieces, str8_struct(&req->id));
str8_list_push(scratch.arena, &pieces, str8_struct(&code));
str8_list_push(scratch.arena, &pieces, str8_struct(&body_size));
str8_list_concat_in_place(&pieces, &req->finished_body_pieces);
U64 data_size = pieces.total_size;
str8_list_push_front(scratch.arena, &pieces, str8_struct(&data_size));
if(data_size + sizeof(U64) > response_data_max_size)
{
U64 overkill = data_size + sizeof(U64) - response_data_max_size;
data_size -= overkill;
}
response_data = str8_list_join(scratch.arena, &pieces, 0);
response_data.size = data_size + sizeof(U64);
}
//- rjf: write response
if(response_is_possible)
{
RingGuard g = guarded_ring_open(req->out_ring);
guarded_ring_write_string_or_wait(&g, response_data, max_U64);
guarded_ring_close(&g);
}
//- rjf: free this request slot
req->active = 0;
arena_clear(req->arena);
MemoryZeroStruct(&req->finished_body_pieces);
@@ -427,8 +465,6 @@ http_async_tick(void)
req->next_body_piece_size = 0;
WinHttpCloseHandle(req->hConnect);
WinHttpCloseHandle(req->hRequest);
scratch_end(scratch);
}
}
}
@@ -485,14 +521,16 @@ http_pop_response(Arena *arena, GuardedRing *out_ring, HTTP_Response *response_o
B32 result = guarded_ring_read_struct_or_wait(&g, &data_size, endt_us);
if(result)
{
U64 data_size_minus_size_prefix = data_size - sizeof(data_size);
String8 data = {0};
data.size = data_size;
data.size = data_size_minus_size_prefix;
data.str = push_array(arena, U8, data.size);
guarded_ring_read_or_wait(&g, data.size, data.str, max_U64);
U64 off = 0;
off += str8_deserial_read_struct(data, off, &response_out->id);
off += str8_deserial_read_struct(data, off, &response_out->has_more);
off += str8_deserial_read_struct(data, off, &response_out->code);
off += str8_deserial_read_struct(data, off, &response_out->body.size);
response_out->body.size = (data.size - off);
response_out->body.str = push_array(arena, U8, response_out->body.size);
off += str8_deserial_read(data, off, response_out->body.str, response_out->body.size, 1);
}
src/win32/http/win32_http.h
+2
View File
@@ -34,6 +34,8 @@ struct W32_HTTP_Request
HINTERNET hRequest;
void *optional; // NOTE(rjf): must persist with in-flight requests
U64 optional_size;
HTTP_StatusCode status_code;
U64 arena_start_body_read_pos;
String8List finished_body_pieces;
void *next_body_piece;
U64 next_body_piece_size;
src/win32/symbol_server/win32_symbol_server.c
+327 -8
View File
@@ -7,6 +7,8 @@ smsv_init(void)
Arena *arena = arena_alloc();
w32_smsv_state = push_array(arena, W32_SMSV_State, 1);
w32_smsv_state->arena = arena;
// rjf: get symbol server configuration from environment
{
// rjf: extract parameterizations from _NT_SYMBOL_PATH
{
@@ -71,24 +73,257 @@ smsv_init(void)
}
}
// rjf fall back to using local appdata temp symbol cache folder
// rjf fall back to using C:/SymbolCache (or wherever windows is located)
if(w32_smsv_state->symbol_cache_path.size == 0)
{
Temp scratch = scratch_begin(0, 0);
U64 size = KB(32);
U16 *buffer = push_array_no_zero(scratch.arena, U16, size);
if(SUCCEEDED(SHGetFolderPathW(0, CSIDL_LOCAL_APPDATA, 0, 0, (WCHAR*)buffer)))
if(SUCCEEDED(SHGetFolderPathW(0, CSIDL_SYSTEM, 0, 0, (WCHAR*)buffer)))
{
String8 local_appdata = str8_from_16(scratch.arena, str16_cstring(buffer));
w32_smsv_state->symbol_cache_path = str8f(arena, "%S\\Temp\\SymbolCache", local_appdata);
String8 system32 = str8_from_16(scratch.arena, str16_cstring(buffer));
String8 containing_drive = str8_prefix(system32, str8_find_needle(system32, 0, s("/"), StringMatchFlag_SlashInsensitive));
w32_smsv_state->symbol_cache_path = str8f(arena, "%S\\SymbolCache", containing_drive);
}
scratch_end(scratch);
}
}
// rjf: set up task cache
{
w32_smsv_state->task_slots_count = 256;
w32_smsv_state->task_slots = push_array(arena, W32_SMSV_TaskSlot, w32_smsv_state->task_slots_count);
w32_smsv_state->task_stripes = stripe_array_alloc(arena);
}
// rjf: set up rings
w32_smsv_state->http_response_ring = guarded_ring_alloc(arena, KB(256));
w32_smsv_state->new_task_ring = guarded_ring_alloc(arena, KB(8));
w32_smsv_state->retry_arena = arena_alloc();
}
internal void
smsv_async_tick(void)
{
Temp scratch = scratch_begin(0, 0);
//- rjf: pop new tasks from new task ring, gather them
W32_SMSV_TaskNode *first_request_task = 0;
W32_SMSV_TaskNode *last_request_task = 0;
U64 request_task_count = 0;
if(lane_idx() == 0)
{
for(B32 done = 0;!done;)
{
// rjf: get the next task ID to start
U64 task_id = 0;
{
RingGuard g = guarded_ring_open(w32_smsv_state->new_task_ring);
if(!guarded_ring_try_read_struct(&g, &task_id))
{
done = 1;
}
guarded_ring_close(&g);
}
// rjf: gather
if(!done)
{
W32_SMSV_Task *task = (W32_SMSV_Task *)task_id;
W32_SMSV_TaskNode *n = push_array(scratch.arena, W32_SMSV_TaskNode, 1);
n->task = task;
SLLQueuePush(first_request_task, last_request_task, n);
request_task_count += 1;
}
}
}
lane_sync();
//- rjf: gather tasks from the retry batch
if(lane_idx() == 0)
{
for(W32_SMSV_TaskNode *t = w32_smsv_state->first_retry_request; t != 0; t = t->next)
{
W32_SMSV_TaskNode *n = push_array(scratch.arena, W32_SMSV_TaskNode, 1);
n->task = t->task;
SLLQueuePush(first_request_task, last_request_task, n);
request_task_count += 1;
}
arena_clear(w32_smsv_state->retry_arena);
w32_smsv_state->first_retry_request = w32_smsv_state->last_retry_request = 0;
}
lane_sync();
//- rjf: pop responses -> for any responses that suggest that a particular debug info
// is not present on the server, we need to generate new request nodes for those tasks,
// to fall back to subsequent server URLS.
W32_SMSV_TaskNode *first_finish_task = 0;
W32_SMSV_TaskNode *last_finish_task = 0;
U64 finish_task_count = 0;
if(lane_idx() == 0)
{
for(HTTP_Response r = {0}; http_pop_response(scratch.arena, w32_smsv_state->http_response_ring, &r, 0);)
{
W32_SMSV_Task *task = (W32_SMSV_Task *)r.id;
if(task != 0)
{
if(r.body.size != 0)
{
str8_list_push(task->arena, &task->file_pieces, str8_copy(task->arena, r.body));
}
if(r.has_more == 0 && task->status != W32_SMSV_TaskStatus_DoneDownloading)
{
task->status = W32_SMSV_TaskStatus_DoneDownloading;
W32_SMSV_TaskNode *n = push_array(scratch.arena, W32_SMSV_TaskNode, 1);
n->task = task;
SLLQueuePush(first_finish_task, last_finish_task, n);
finish_task_count += 1;
}
}
}
}
lane_sync();
//- rjf: flatten all request tasks
W32_SMSV_Task **request_tasks = 0;
if(lane_idx() == 0)
{
request_tasks = push_array(scratch.arena, W32_SMSV_Task *, request_task_count);
{
U64 idx = 0;
for EachNode(n, W32_SMSV_TaskNode, first_request_task)
{
request_tasks[idx] = n->task;
idx += 1;
}
}
}
lane_sync_u64(&request_tasks, 0);
//- rjf: do all request tasks
{
Rng1U64 range = lane_range(request_task_count);
for EachInRange(idx, range)
{
W32_SMSV_Task *task = request_tasks[idx];
// rjf: adjust the server URL node that we're using for this task
if(task->status == W32_SMSV_TaskStatus_Failed || task->status == W32_SMSV_TaskStatus_Null)
{
if(task->current_server_url_node == 0)
{
task->current_server_url_node = w32_smsv_state->symbol_server_urls.first;
}
else
{
task->current_server_url_node = task->current_server_url_node->next;
}
}
// rjf: get server URL
String8 server_url = {0};
if(task->current_server_url_node != 0)
{
server_url = task->current_server_url_node->string;
}
// rjf: kick off request if we have a server URL
if(server_url.size != 0)
{
task->status = W32_SMSV_TaskStatus_Requested;
Guid guid = task->guid;
U64 age = task->age;
String8 debug_info_unique_identifier = {0};
{
debug_info_unique_identifier = str8f(scratch.arena, "%08X%04X%04X%02X%02X%02X%02X%02X%02X%02X%02X%I64x",
guid.data1, guid.data2, guid.data3, guid.data4[0], guid.data4[1], guid.data4[2], guid.data4[3], guid.data4[4], guid.data4[5], guid.data4[6], guid.data4[7], age);
}
HTTP_RequestParams params =
{
.id = (U64)task,
.method = HTTP_Method_Get,
.url = str8f(scratch.arena, "%S/%S/%S/%S", server_url, task->dbg_name, debug_info_unique_identifier, task->dbg_name),
};
if(!http_push_request(w32_smsv_state->http_response_ring, &params, 0))
{
W32_SMSV_TaskNode *n = push_array(w32_smsv_state->retry_arena, W32_SMSV_TaskNode, 1);
SLLQueuePush(w32_smsv_state->first_retry_request, w32_smsv_state->last_retry_request, n);
n->task = task;
ins_atomic_u32_eval_assign(&async_loop_again, 1);
cond_var_broadcast(async_tick_start_cond_var);
}
}
// rjf: if we don't have a server URL, mark this task as failed
else
{
task->status = W32_SMSV_TaskStatus_Failed;
W32_SMSV_TaskNode *n = push_array(scratch.arena, W32_SMSV_TaskNode, 1);
n->task = task;
SLLQueuePush(first_finish_task, last_finish_task, n);
finish_task_count += 1;
}
}
}
//- rjf: flatten all finish tasks
W32_SMSV_Task **finish_tasks = 0;
if(lane_idx() == 0)
{
finish_tasks = push_array(scratch.arena, W32_SMSV_Task *, finish_task_count);
{
U64 idx = 0;
for EachNode(n, W32_SMSV_TaskNode, first_finish_task)
{
finish_tasks[idx] = n->task;
idx += 1;
}
}
}
lane_sync_u64(&finish_tasks, 0);
//- rjf: do all finish tasks
{
Rng1U64 range = lane_range(finish_task_count);
for EachInRange(idx, range)
{
W32_SMSV_Task *task = finish_tasks[idx];
// rjf: successful task -> write to local path
if(task->status != W32_SMSV_TaskStatus_Failed)
{
String8 directory = str8_chop_last_slash(task->local_path);
for(U64 slash_pos = 0; slash_pos <= directory.size;)
{
slash_pos = str8_find_needle(directory, slash_pos+1, s("/"), StringMatchFlag_SlashInsensitive);
String8 root_dir = str8_substr(directory, r1u64(0, slash_pos));
make_directory(root_dir);
if(slash_pos == directory.size)
{
break;
}
}
write_data_list_to_file_path(task->local_path, task->file_pieces);
}
// rjf: evict task from cache
U64 hash = u64_hash_from_str8(task->local_path);
U64 slot_idx = hash%w32_smsv_state->task_slots_count;
W32_SMSV_TaskSlot *slot = &w32_smsv_state->task_slots[slot_idx];
Stripe *stripe = stripe_from_slot_idx(&w32_smsv_state->task_stripes, slot_idx);
RWMutexScope(stripe->rw_mutex, 1)
{
DLLRemove(slot->first, slot->last, task);
arena_release(task->arena);
}
}
}
scratch_end(scratch);
}
internal String8
smsv_local_path_from_key(Arena *arena, String8 module_name, String8 dbg_name, Guid guid, U64 age)
smsv_local_path_from_key(Arena *arena, String8 dbg_name, Guid guid, U64 age)
{
String8 result = {0};
Temp scratch = scratch_begin(&arena, 1);
@@ -96,19 +331,103 @@ smsv_local_path_from_key(Arena *arena, String8 module_name, String8 dbg_name, Gu
// rjf: form unique identifier based on guid
String8 debug_info_unique_identifier = {0};
{
debug_info_unique_identifier = str8f(scratch.arena, "%08X%04X%04X%02X%02X%02X%02X%02X%02X%02X%02X%x",
debug_info_unique_identifier = str8f(scratch.arena, "%08X%04X%04X%02X%02X%02X%02X%02X%02X%02X%02X%I64x",
guid.data1, guid.data2, guid.data3, guid.data4[0], guid.data4[1], guid.data4[2], guid.data4[3], guid.data4[4], guid.data4[5], guid.data4[6], guid.data4[7], age);
}
// rjf: form full path
if(debug_info_unique_identifier.size != 0)
{
// TODO(rjf): assuming 'stripped' for now
String8 symbol_cache_path = w32_smsv_state->symbol_cache_path;
String8 path = str8f(scratch.arena, "%S/%S/%S/stripped/%S", symbol_cache_path, module_name, debug_info_unique_identifier, dbg_name);
String8 path = str8f(scratch.arena, "%S/%S/%S/%S", symbol_cache_path, dbg_name, debug_info_unique_identifier, dbg_name);
result = path_normalized_from_string(arena, path);
}
// rjf: determine if we already have this debug info stored locally
B32 already_cached_locally = (properties_from_file_path(result).modified != 0);
// rjf: if not cached: record (local path -> download task) mapping
B32 task_is_new = 0;
U64 task_id = 0;
if(!already_cached_locally)
{
U64 hash = u64_hash_from_str8(result);
U64 slot_idx = hash%w32_smsv_state->task_slots_count;
W32_SMSV_TaskSlot *slot = &w32_smsv_state->task_slots[slot_idx];
Stripe *stripe = stripe_from_slot_idx(&w32_smsv_state->task_stripes, slot_idx);
for(B32 write_mode = 0; write_mode <= 1; write_mode += 1)
{
B32 already_exists = 0;
RWMutexScope(stripe->rw_mutex, write_mode)
{
W32_SMSV_Task *node = 0;
for(W32_SMSV_Task *n = slot->first; n != 0; n = n->next)
{
if(MemoryMatchStruct(&n->guid, &guid))
{
already_exists = 1;
node = n;
break;
}
}
if(node == 0 && write_mode)
{
Arena *arena = arena_alloc();
node = push_array(arena, W32_SMSV_Task, 1);
node->arena = arena;
node->local_path = str8_copy(arena, result);
node->dbg_name = str8_copy(arena, dbg_name);
node->guid = guid;
node->age = age;
DLLPushBack(slot->first, slot->last, node);
task_is_new = 1;
task_id = (U64)node;
}
}
if(already_exists)
{
break;
}
}
}
// rjf: if the task is new -> push request to start task
if(task_is_new)
{
RingGuard g = guarded_ring_open(w32_smsv_state->new_task_ring);
guarded_ring_write_struct_or_wait(&g, &task_id, max_U64);
guarded_ring_close(&g);
ins_atomic_u32_eval_assign(&async_loop_again, 1);
cond_var_broadcast(async_tick_start_cond_var);
}
}
scratch_end(scratch);
return result;
}
internal SMSV_Status
smsv_status_from_local_path(String8 path)
{
SMSV_Status status = SMSV_Status_Null;
{
Temp scratch = scratch_begin(0, 0);
String8 path_normalized = path_normalized_from_string(scratch.arena, path);
U64 hash = u64_hash_from_str8(path_normalized);
U64 slot_idx = hash%w32_smsv_state->task_slots_count;
W32_SMSV_TaskSlot *slot = &w32_smsv_state->task_slots[slot_idx];
Stripe *stripe = stripe_from_slot_idx(&w32_smsv_state->task_stripes, slot_idx);
RWMutexScope(stripe->rw_mutex, 0)
{
for(W32_SMSV_Task *n = slot->first; n != 0; n = n->next)
{
if(str8_match(n->local_path, path, 0))
{
status = SMSV_Status_Pending;
break;
}
}
}
scratch_end(scratch);
}
return status;
}
src/win32/symbol_server/win32_symbol_server.h
+50
View File
@@ -4,12 +4,62 @@
#ifndef WIN32_SYMBOL_SERVER_H
#define WIN32_SYMBOL_SERVER_H
typedef enum W32_SMSV_TaskStatus
{
W32_SMSV_TaskStatus_Null,
W32_SMSV_TaskStatus_Requested,
W32_SMSV_TaskStatus_DoneDownloading,
W32_SMSV_TaskStatus_Failed,
}
W32_SMSV_TaskStatus;
typedef struct W32_SMSV_Task W32_SMSV_Task;
struct W32_SMSV_Task
{
W32_SMSV_Task *next;
W32_SMSV_Task *prev;
// rjf: key / parameters
Arena *arena;
String8 local_path;
String8 dbg_name;
Guid guid;
U64 age;
// rjf: working state
W32_SMSV_TaskStatus status;
String8Node *current_server_url_node;
String8List file_pieces;
};
typedef struct W32_SMSV_TaskNode W32_SMSV_TaskNode;
struct W32_SMSV_TaskNode
{
W32_SMSV_TaskNode *next;
W32_SMSV_Task *task;
};
typedef struct W32_SMSV_TaskSlot W32_SMSV_TaskSlot;
struct W32_SMSV_TaskSlot
{
W32_SMSV_Task *first;
W32_SMSV_Task *last;
};
typedef struct W32_SMSV_State W32_SMSV_State;
struct W32_SMSV_State
{
Arena *arena;
String8 symbol_cache_path;
String8List symbol_server_urls;
U64 task_slots_count;
W32_SMSV_TaskSlot *task_slots;
StripeArray task_stripes;
GuardedRing *http_response_ring;
GuardedRing *new_task_ring;
Arena *retry_arena;
W32_SMSV_TaskNode *first_retry_request;
W32_SMSV_TaskNode *last_retry_request;
};
global W32_SMSV_State *w32_smsv_state = 0;