stub out linked list view rule in new eval viz system; sketch out pointer graph cache layer, for pointer chasing & building graphs asynchronously, for use in the linked list viewer & other future visualizers

This commit is contained in:
Ryan Fleury
2024-10-13 20:07:39 -07:00
parent 000970758a
commit adb8d5237c
7 changed files with 487 additions and 13 deletions
+13 -12
View File
@@ -88,20 +88,21 @@
// For any view rules in this layer which also have graphical features, they
// are specified in both tables under the same name.
@table(coverage_check name name_lower string ih ex xr xe vb display_name docs schema description)
@table(coverage_check name name_lower string ih ex xr xe display_name docs schema description)
EV_ViewRuleTable:
{
{x Default default "default" - - - x x "Default" - "" "" }
{x Array array "array" - - x - - "Array" x "x:{expr}" "Specifies that a pointer points to N elements, rather than only 1." }
{x Slice slice "slice" - - x - - "Slice" x "" "Specifies that a pointer within a struct, also containing an integer, points to the number of elements encoded by the integer." }
{x ByteSwap bswap "bswap" x - x - - "Byte Swap" x "" "Specifies that all integral evaluations should be byte-swapped, such that their endianness is reversed." }
{x Cast cast "cast" - - x - - "Cast" x "x:{type}" "Specifies that the expression to which the view rule is applied should be casted to the provided type." }
{x Only only "only" x - x - - "Only" x "" "Specifies that only the provided member names should be shown in user-defined-type expansions." }
{x Omit omit "omit" x - x - - "Omit" x "" "Specifies that the provided member names should not be shown in user-defined-type expansions." }
{x Bin bin "bin" x - - - - "Display In Binary" x "" "Specifies that all numeric values should be shown in base 2 (binary)." }
{x Oct oct "oct" x - - - - "Display In Octal" x "" "Specifies that all numeric values should be shown in base 8 (octal)." }
{x Dec dec "dec" x - - - - "Display In Decimal" x "" "Specifies that all numeric values should be shown in base 10 (decimal)." }
{x Hex hex "hex" x - - - - "Display In Hexadecimal" x "" "Specifies that all numeric values should be shown in base 16 (hexadecimal)." }
{x Default default "default" - - - x "Default" - "" "" }
{x Array array "array" - - x - "Array" x "x:{expr}" "Specifies that a pointer points to N elements, rather than only 1." }
{x List list "list" - - - x "Array" x "x:{expr}" "Specifies that a pointer points to N elements, rather than only 1." }
{x Slice slice "slice" - - x - "Slice" x "" "Specifies that a pointer within a struct, also containing an integer, points to the number of elements encoded by the integer." }
{x ByteSwap bswap "bswap" x - x - "Byte Swap" x "" "Specifies that all integral evaluations should be byte-swapped, such that their endianness is reversed." }
{x Cast cast "cast" - - x - "Cast" x "x:{type}" "Specifies that the expression to which the view rule is applied should be casted to the provided type." }
{x Only only "only" x - x - "Only" x "" "Specifies that only the provided member names should be shown in user-defined-type expansions." }
{x Omit omit "omit" x - x - "Omit" x "" "Specifies that the provided member names should not be shown in user-defined-type expansions." }
{x Bin bin "bin" x - - - "Display In Binary" x "" "Specifies that all numeric values should be shown in base 2 (binary)." }
{x Oct oct "oct" x - - - "Display In Octal" x "" "Specifies that all numeric values should be shown in base 8 (octal)." }
{x Dec dec "dec" x - - - "Display In Decimal" x "" "Specifies that all numeric values should be shown in base 10 (decimal)." }
{x Hex hex "hex" x - - - "Display In Hexadecimal" x "" "Specifies that all numeric values should be shown in base 16 (hexadecimal)." }
}
@enum EV_ViewRuleKind:
@@ -287,6 +287,21 @@ EV_VIEW_RULE_EXPR_RESOLUTION_FUNCTION_DEF(array)
return expr;
}
////////////////////////////////
//~ rjf: "list"
EV_VIEW_RULE_EXPR_EXPAND_INFO_FUNCTION_DEF(list)
{
EV_ExpandInfo info = {0};
return info;
}
EV_VIEW_RULE_EXPR_EXPAND_RANGE_INFO_FUNCTION_DEF(list)
{
EV_ExpandRangeInfo info = {0};
return info;
}
////////////////////////////////
//~ rjf: "slice"
@@ -4,10 +4,11 @@
//- GENERATED CODE
C_LINKAGE_BEGIN
EV_ViewRuleInfo ev_builtin_view_rule_info_table[11] =
EV_ViewRuleInfo ev_builtin_view_rule_info_table[12] =
{
{str8_lit_comp("default"), (EV_ViewRuleInfoFlag_Inherited*0)|(EV_ViewRuleInfoFlag_Expandable*0), EV_VIEW_RULE_EXPR_RESOLUTION_FUNCTION_NAME(identity), EV_VIEW_RULE_EXPR_EXPAND_INFO_FUNCTION_NAME(default) , EV_VIEW_RULE_EXPR_EXPAND_RANGE_INFO_FUNCTION_NAME(default) , EV_VIEW_RULE_EXPR_EXPAND_ID_FROM_NUM_FUNCTION_NAME(identity), EV_VIEW_RULE_EXPR_EXPAND_NUM_FROM_ID_FUNCTION_NAME(identity) },
{str8_lit_comp("array"), (EV_ViewRuleInfoFlag_Inherited*0)|(EV_ViewRuleInfoFlag_Expandable*0), EV_VIEW_RULE_EXPR_RESOLUTION_FUNCTION_NAME(array) , EV_VIEW_RULE_EXPR_EXPAND_INFO_FUNCTION_NAME(nil), EV_VIEW_RULE_EXPR_EXPAND_RANGE_INFO_FUNCTION_NAME(nil), EV_VIEW_RULE_EXPR_EXPAND_ID_FROM_NUM_FUNCTION_NAME(identity), EV_VIEW_RULE_EXPR_EXPAND_NUM_FROM_ID_FUNCTION_NAME(identity) },
{str8_lit_comp("list"), (EV_ViewRuleInfoFlag_Inherited*0)|(EV_ViewRuleInfoFlag_Expandable*0), EV_VIEW_RULE_EXPR_RESOLUTION_FUNCTION_NAME(identity), EV_VIEW_RULE_EXPR_EXPAND_INFO_FUNCTION_NAME(list) , EV_VIEW_RULE_EXPR_EXPAND_RANGE_INFO_FUNCTION_NAME(list) , EV_VIEW_RULE_EXPR_EXPAND_ID_FROM_NUM_FUNCTION_NAME(identity), EV_VIEW_RULE_EXPR_EXPAND_NUM_FROM_ID_FUNCTION_NAME(identity) },
{str8_lit_comp("slice"), (EV_ViewRuleInfoFlag_Inherited*0)|(EV_ViewRuleInfoFlag_Expandable*0), EV_VIEW_RULE_EXPR_RESOLUTION_FUNCTION_NAME(slice) , EV_VIEW_RULE_EXPR_EXPAND_INFO_FUNCTION_NAME(nil), EV_VIEW_RULE_EXPR_EXPAND_RANGE_INFO_FUNCTION_NAME(nil), EV_VIEW_RULE_EXPR_EXPAND_ID_FROM_NUM_FUNCTION_NAME(identity), EV_VIEW_RULE_EXPR_EXPAND_NUM_FROM_ID_FUNCTION_NAME(identity) },
{str8_lit_comp("bswap"), (EV_ViewRuleInfoFlag_Inherited*1)|(EV_ViewRuleInfoFlag_Expandable*0), EV_VIEW_RULE_EXPR_RESOLUTION_FUNCTION_NAME(bswap) , EV_VIEW_RULE_EXPR_EXPAND_INFO_FUNCTION_NAME(nil), EV_VIEW_RULE_EXPR_EXPAND_RANGE_INFO_FUNCTION_NAME(nil), EV_VIEW_RULE_EXPR_EXPAND_ID_FROM_NUM_FUNCTION_NAME(identity), EV_VIEW_RULE_EXPR_EXPAND_NUM_FROM_ID_FUNCTION_NAME(identity) },
{str8_lit_comp("cast"), (EV_ViewRuleInfoFlag_Inherited*0)|(EV_ViewRuleInfoFlag_Expandable*0), EV_VIEW_RULE_EXPR_RESOLUTION_FUNCTION_NAME(cast) , EV_VIEW_RULE_EXPR_EXPAND_INFO_FUNCTION_NAME(nil), EV_VIEW_RULE_EXPR_EXPAND_RANGE_INFO_FUNCTION_NAME(nil), EV_VIEW_RULE_EXPR_EXPAND_ID_FROM_NUM_FUNCTION_NAME(identity), EV_VIEW_RULE_EXPR_EXPAND_NUM_FROM_ID_FUNCTION_NAME(identity) },
@@ -10,6 +10,7 @@ typedef enum EV_ViewRuleKind
{
EV_ViewRuleKind_Default,
EV_ViewRuleKind_Array,
EV_ViewRuleKind_List,
EV_ViewRuleKind_Slice,
EV_ViewRuleKind_ByteSwap,
EV_ViewRuleKind_Cast,
@@ -29,5 +30,7 @@ EV_VIEW_RULE_EXPR_RESOLUTION_FUNCTION_DEF(cast);
EV_VIEW_RULE_EXPR_RESOLUTION_FUNCTION_DEF(only);
EV_VIEW_RULE_EXPR_RESOLUTION_FUNCTION_DEF(omit);
EV_VIEW_RULE_EXPR_EXPAND_INFO_FUNCTION_DEF(default);
EV_VIEW_RULE_EXPR_EXPAND_INFO_FUNCTION_DEF(list);
EV_VIEW_RULE_EXPR_EXPAND_RANGE_INFO_FUNCTION_DEF(default);
EV_VIEW_RULE_EXPR_EXPAND_RANGE_INFO_FUNCTION_DEF(list);
#endif // EVAL_VISUALIZATION_META_H
+268
View File
@@ -0,0 +1,268 @@
// Copyright (c) 2024 Epic Games Tools
// Licensed under the MIT license (https://opensource.org/license/mit/)
////////////////////////////////
//~ rjf: Main Layer Initialization
internal void
ptg_init(void)
{
Arena *arena = arena_alloc();
ptg_shared = push_array(arena, PTG_Shared, 1);
ptg_shared->arena = arena;
ptg_shared->slots_count = 1024;
ptg_shared->stripes_count = Min(ptg_shared->slots_count, os_get_system_info()->logical_processor_count);
ptg_shared->slots = push_array(arena, PTG_GraphSlot, ptg_shared->slots_count);
ptg_shared->stripes = push_array(arena, PTG_GraphStripe, ptg_shared->stripes_count);
for(U64 idx = 0; idx < ptg_shared->stripes_count; idx += 1)
{
ptg_shared->stripes[idx].arena = arena_alloc();
ptg_shared->stripes[idx].rw_mutex = os_rw_mutex_alloc();
ptg_shared->stripes[idx].cv = os_condition_variable_alloc();
}
ptg_shared->u2b_ring_size = KB(64);
ptg_shared->u2b_ring_base = push_array_no_zero(arena, U8, ptg_shared->u2b_ring_size);
ptg_shared->u2b_ring_cv = os_condition_variable_alloc();
ptg_shared->u2b_ring_mutex = os_mutex_alloc();
ptg_shared->builder_thread_count = Clamp(1, os_get_system_info()->logical_processor_count-1, 4);
ptg_shared->builder_threads = push_array(arena, OS_Handle, ptg_shared->builder_thread_count);
for(U64 idx = 0; idx < ptg_shared->builder_thread_count; idx += 1)
{
ptg_shared->builder_threads[idx] = os_thread_launch(ptg_builder_thread__entry_point, (void *)idx, 0);
}
ptg_shared->evictor_thread = os_thread_launch(ptg_evictor_thread__entry_point, 0, 0);
}
////////////////////////////////
//~ rjf: User Clock
internal void
ptg_user_clock_tick(void)
{
ins_atomic_u64_inc_eval(&ptg_shared->user_clock_idx);
}
internal U64
ptg_user_clock_idx(void)
{
return ins_atomic_u64_eval(&ptg_shared->user_clock_idx);
}
////////////////////////////////
//~ rjf: Scoped Access
internal PTG_Scope *
ptg_scope_open(void)
{
if(ptg_tctx == 0)
{
Arena *arena = arena_alloc();
ptg_tctx = push_array(arena, PTG_TCTX, 1);
ptg_tctx->arena = arena;
}
PTG_Scope *scope = ptg_tctx->free_scope;
if(scope)
{
SLLStackPop(ptg_tctx->free_scope);
}
else
{
scope = push_array_no_zero(ptg_tctx->arena, PTG_Scope, 1);
}
MemoryZeroStruct(scope);
return scope;
}
internal void
ptg_scope_close(PTG_Scope *scope)
{
for(PTG_Touch *touch = scope->top_touch, *next = 0; touch != 0; touch = next)
{
next = touch->next;
ins_atomic_u64_dec_eval(&touch->node->scope_ref_count);
SLLStackPush(ptg_tctx->free_touch, touch);
}
SLLStackPush(ptg_tctx->free_scope, scope);
}
internal void
ptg_scope_touch_node__stripe_r_guarded(PTG_Scope *scope, PTG_GraphNode *node)
{
PTG_Touch *touch = ptg_tctx->free_touch;
ins_atomic_u64_inc_eval(&node->scope_ref_count);
ins_atomic_u64_eval_assign(&node->last_time_touched_us, os_now_microseconds());
ins_atomic_u64_eval_assign(&node->last_user_clock_idx_touched, ptg_user_clock_idx());
if(touch != 0)
{
SLLStackPop(ptg_tctx->free_touch);
}
else
{
touch = push_array_no_zero(ptg_tctx->arena, PTG_Touch, 1);
}
MemoryZeroStruct(touch);
touch->node = node;
SLLStackPush(scope->top_touch, touch);
}
////////////////////////////////
//~ rjf: Cache Lookups
internal PTG_Graph *
ptg_graph_from_key(PTG_Scope *scope, PTG_Key *key)
{
}
////////////////////////////////
//~ rjf: Transfer Threads
internal B32
ptg_u2b_enqueue_req(PTG_Key *key, U64 endt_us)
{
B32 good = 0;
OS_MutexScope(ptg_shared->u2b_ring_mutex) for(;;)
{
U64 unconsumed_size = ptg_shared->u2b_ring_write_pos-ptg_shared->u2b_ring_read_pos;
U64 available_size = ptg_shared->u2b_ring_size-unconsumed_size;
if(available_size >= sizeof(key))
{
good = 1;
ptg_shared->u2b_ring_write_pos += ring_write_struct(ptg_shared->u2b_ring_base, ptg_shared->u2b_ring_size, ptg_shared->u2b_ring_write_pos, &key);
break;
}
if(os_now_microseconds() >= endt_us)
{
break;
}
os_condition_variable_wait(ptg_shared->u2b_ring_cv, ptg_shared->u2b_ring_mutex, endt_us);
}
if(good)
{
os_condition_variable_broadcast(ptg_shared->u2b_ring_cv);
}
return good;
}
internal void
ptg_u2b_dequeue_req(PTG_Key *key_out)
{
OS_MutexScope(ptg_shared->u2b_ring_mutex) for(;;)
{
U64 unconsumed_size = ptg_shared->u2b_ring_write_pos-ptg_shared->u2b_ring_read_pos;
if(unconsumed_size >= sizeof(*key_out))
{
ptg_shared->u2b_ring_read_pos += ring_read_struct(ptg_shared->u2b_ring_base, ptg_shared->u2b_ring_size, ptg_shared->u2b_ring_read_pos, key_out);
break;
}
os_condition_variable_wait(ptg_shared->u2b_ring_cv, ptg_shared->u2b_ring_mutex, max_U64);
}
os_condition_variable_broadcast(ptg_shared->u2b_ring_cv);
}
internal void
ptg_builder_thread__entry_point(void *p)
{
for(;;)
{
HS_Scope *scope = hs_scope_open();
//- rjf: get next key
PTG_Key key = {0};
ptg_u2b_dequeue_req(&key);
//- rjf: unpack hash
U64 slot_idx = key.root_hash.u64[1]%ptg_shared->slots_count;
U64 stripe_idx = slot_idx%ptg_shared->stripes_count;
PTG_GraphSlot *slot = &ptg_shared->slots[slot_idx];
PTG_GraphStripe *stripe = &ptg_shared->stripes[stripe_idx];
//- rjf: take task
B32 got_task = 0;
OS_MutexScopeR(stripe->rw_mutex)
{
for(PTG_GraphNode *n = slot->first; n != 0; n = n->next)
{
if(MemoryMatchStruct(&n->key, &key))
{
got_task = !ins_atomic_u32_eval_cond_assign(&n->is_working, 1, 0);
break;
}
}
}
//- rjf: commit results to cache
if(got_task) OS_MutexScopeW(stripe->rw_mutex)
{
for(PTG_GraphNode *n = slot->first; n != 0; n = n->next)
{
if(MemoryMatchStruct(&n->key, &key))
{
ins_atomic_u32_eval_assign(&n->is_working, 0);
ins_atomic_u64_inc_eval(&n->load_count);
break;
}
}
}
hs_scope_close(scope);
}
}
////////////////////////////////
//~ rjf: Evictor Threads
internal void
ptg_evictor_thread__entry_point(void *p)
{
for(;;)
{
U64 check_time_us = os_now_microseconds();
U64 check_time_user_clocks = ptg_user_clock_idx();
U64 evict_threshold_us = 10*1000000;
U64 evict_threshold_user_clocks = 10;
for(U64 slot_idx = 0; slot_idx < ptg_shared->slots_count; slot_idx += 1)
{
U64 stripe_idx = slot_idx%ptg_shared->stripes_count;
PTG_GraphSlot *slot = &ptg_shared->slots[slot_idx];
PTG_GraphStripe *stripe = &ptg_shared->stripes[stripe_idx];
B32 slot_has_work = 0;
OS_MutexScopeR(stripe->rw_mutex)
{
for(PTG_GraphNode *n = slot->first; n != 0; n = n->next)
{
if(n->scope_ref_count == 0 &&
n->last_time_touched_us+evict_threshold_us <= check_time_us &&
n->last_user_clock_idx_touched+evict_threshold_user_clocks <= check_time_user_clocks &&
n->load_count != 0 &&
n->is_working == 0)
{
slot_has_work = 1;
break;
}
}
}
if(slot_has_work) OS_MutexScopeW(stripe->rw_mutex)
{
for(PTG_GraphNode *n = slot->first, *next = 0; n != 0; n = next)
{
next = n->next;
if(n->scope_ref_count == 0 &&
n->last_time_touched_us+evict_threshold_us <= check_time_us &&
n->last_user_clock_idx_touched+evict_threshold_user_clocks <= check_time_user_clocks &&
n->load_count != 0 &&
n->is_working == 0)
{
DLLRemove(slot->first, slot->last, n);
arena_clear(n->node_arena);
arena_clear(n->link_arena);
SLLStackPush(stripe->free_node, n);
}
}
}
os_sleep_milliseconds(5);
}
os_sleep_milliseconds(1000);
}
}
+184
View File
@@ -0,0 +1,184 @@
// Copyright (c) 2024 Epic Games Tools
// Licensed under the MIT license (https://opensource.org/license/mit/)
#ifndef PTR_GRAPH_CACHE_H
#define PTR_GRAPH_CACHE_H
////////////////////////////////
//~ rjf: Graph Search Key
typedef struct PTG_Key PTG_Key;
struct PTG_Key
{
U128 root_hash;
U64 link_offsets[8];
U64 link_offsets_count;
};
////////////////////////////////
//~ rjf: Cache Types
typedef struct PTG_Node PTG_Node;
struct PTG_Node
{
U64 value;
};
typedef struct PTG_Link PTG_Link;
struct PTG_Link
{
U32 from;
U32 to;
};
typedef struct PTG_Graph PTG_Graph;
struct PTG_Graph
{
PTG_Node *nodes;
U64 nodes_count;
PTG_Link *links;
U64 links_count;
};
typedef struct PTG_GraphNode PTG_GraphNode;
struct PTG_GraphNode
{
// rjf: links
PTG_GraphNode *next;
PTG_GraphNode *prev;
// rjf: key
PTG_Key key;
// rjf: metadata
U64 scope_ref_count;
U64 last_time_touched_us;
U64 last_user_clock_idx_touched;
U64 load_count;
B32 is_working;
// rjf: content
Arena *node_arena;
Arena *link_arena;
PTG_Graph graph;
};
typedef struct PTG_GraphSlot PTG_GraphSlot;
struct PTG_GraphSlot
{
PTG_GraphNode *first;
PTG_GraphNode *last;
};
typedef struct PTG_GraphStripe PTG_GraphStripe;
struct PTG_GraphStripe
{
Arena *arena;
OS_Handle rw_mutex;
OS_Handle cv;
PTG_GraphNode *free_node;
};
////////////////////////////////
//~ rjf: Scoped Access Types
typedef struct PTG_Touch PTG_Touch;
struct PTG_Touch
{
PTG_Touch *next;
PTG_GraphNode *node;
};
typedef struct PTG_Scope PTG_Scope;
struct PTG_Scope
{
PTG_Scope *next;
PTG_Touch *top_touch;
};
////////////////////////////////
//~ rjf: Thread Context
typedef struct PTG_TCTX PTG_TCTX;
struct PTG_TCTX
{
Arena *arena;
PTG_Scope *free_scope;
PTG_Touch *free_touch;
};
////////////////////////////////
//~ rjf: Shared State
typedef struct PTG_Shared PTG_Shared;
struct PTG_Shared
{
Arena *arena;
// rjf: user clock
U64 user_clock_idx;
// rjf: cache
U64 slots_count;
U64 stripes_count;
PTG_GraphSlot *slots;
PTG_GraphStripe *stripes;
// rjf: user -> xfer thread
U64 u2b_ring_size;
U8 *u2b_ring_base;
U64 u2b_ring_write_pos;
U64 u2b_ring_read_pos;
OS_Handle u2b_ring_cv;
OS_Handle u2b_ring_mutex;
// rjf: builder threads
U64 builder_thread_count;
OS_Handle *builder_threads;
// rjf: evictor thread
OS_Handle evictor_thread;
};
////////////////////////////////
//~ rjf: Globals
thread_static PTG_TCTX *ptg_tctx = 0;
global PTG_Shared *ptg_shared = 0;
////////////////////////////////
//~ rjf: Main Layer Initialization
internal void ptg_init(void);
////////////////////////////////
//~ rjf: User Clock
internal void ptg_user_clock_tick(void);
internal U64 ptg_user_clock_idx(void);
////////////////////////////////
//~ rjf: Scoped Access
internal PTG_Scope *ptg_scope_open(void);
internal void ptg_scope_close(PTG_Scope *scope);
internal void ptg_scope_touch_node__stripe_r_guarded(PTG_Scope *scope, PTG_GraphNode *node);
////////////////////////////////
//~ rjf: Cache Lookups
internal PTG_Graph *ptg_graph_from_key(PTG_Scope *scope, PTG_Key *key);
////////////////////////////////
//~ rjf: Transfer Threads
internal B32 ptg_u2b_enqueue_req(PTG_Key *key, U64 endt_us);
internal void ptg_u2b_dequeue_req(PTG_Key *key_out);
internal void ptg_builder_thread__entry_point(void *p);
////////////////////////////////
//~ rjf: Evictor Threads
internal void ptg_evictor_thread__entry_point(void *p);
#endif // PTR_GRAPH_CACHE_H
+2
View File
@@ -581,6 +581,7 @@
#include "ctrl/ctrl_inc.h"
#include "font_provider/font_provider_inc.h"
#include "render/render_inc.h"
#include "ptr_graph_cache/ptr_graph_cache.h"
#include "texture_cache/texture_cache.h"
#include "geo_cache/geo_cache.h"
#include "font_cache/font_cache.h"
@@ -620,6 +621,7 @@
#include "ctrl/ctrl_inc.c"
#include "font_provider/font_provider_inc.c"
#include "render/render_inc.c"
#include "ptr_graph_cache/ptr_graph_cache.c"
#include "texture_cache/texture_cache.c"
#include "geo_cache/geo_cache.c"
#include "font_cache/font_cache.c"