From 4027338a2f9c0fec6d00ea90a4120da97b7126bb Mon Sep 17 00:00:00 2001 From: Ryan Fleury Date: Tue, 26 Mar 2024 11:30:47 -0700 Subject: [PATCH] adjust stripe counts --- src/dbgi/dbgi.c | 6 +++--- src/file_stream/file_stream.c | 34 ++++++++++++++++++++++++++++++- src/file_stream/file_stream.h | 8 ++++++++ src/geo_cache/geo_cache.c | 4 ++-- src/hash_store/hash_store.c | 4 ++-- src/task_system/task_system.c | 2 +- src/text_cache/text_cache.c | 4 ++-- src/texture_cache/texture_cache.c | 4 ++-- 8 files changed, 53 insertions(+), 13 deletions(-) diff --git a/src/dbgi/dbgi.c b/src/dbgi/dbgi.c index 8746bf26..770c5da9 100644 --- a/src/dbgi/dbgi.c +++ b/src/dbgi/dbgi.c @@ -11,7 +11,7 @@ dbgi_init(void) dbgi_shared = push_array(arena, DBGI_Shared, 1); dbgi_shared->arena = arena; dbgi_shared->force_slots_count = 1024; - dbgi_shared->force_stripes_count = 64; + dbgi_shared->force_stripes_count = Min(dbgi_shared->force_slots_count, os_logical_core_count()); dbgi_shared->force_slots = push_array(arena, DBGI_ForceSlot, dbgi_shared->force_slots_count); dbgi_shared->force_stripes = push_array(arena, DBGI_ForceStripe, dbgi_shared->force_stripes_count); for(U64 idx = 0; idx < dbgi_shared->force_stripes_count; idx += 1) @@ -21,7 +21,7 @@ dbgi_init(void) dbgi_shared->force_stripes[idx].cv = os_condition_variable_alloc(); } dbgi_shared->binary_slots_count = 1024; - dbgi_shared->binary_stripes_count = 64; + dbgi_shared->binary_stripes_count = Min(dbgi_shared->binary_slots_count, os_logical_core_count()); dbgi_shared->binary_slots = push_array(arena, DBGI_BinarySlot, dbgi_shared->binary_slots_count); dbgi_shared->binary_stripes = push_array(arena, DBGI_BinaryStripe, dbgi_shared->binary_stripes_count); for(U64 idx = 0; idx < dbgi_shared->binary_stripes_count; idx += 1) @@ -31,7 +31,7 @@ dbgi_init(void) dbgi_shared->binary_stripes[idx].cv = os_condition_variable_alloc(); } dbgi_shared->fuzzy_search_slots_count = 64; - dbgi_shared->fuzzy_search_stripes_count = 8; + dbgi_shared->fuzzy_search_stripes_count = Min(dbgi_shared->fuzzy_search_slots_count, os_logical_core_count()); dbgi_shared->fuzzy_search_slots = push_array(arena, DBGI_FuzzySearchSlot, dbgi_shared->fuzzy_search_slots_count); dbgi_shared->fuzzy_search_stripes = push_array(arena, DBGI_FuzzySearchStripe, dbgi_shared->fuzzy_search_stripes_count); for(U64 idx = 0; idx < dbgi_shared->fuzzy_search_stripes_count; idx += 1) diff --git a/src/file_stream/file_stream.c b/src/file_stream/file_stream.c index ddfa92ed..cc6f025d 100644 --- a/src/file_stream/file_stream.c +++ b/src/file_stream/file_stream.c @@ -11,7 +11,7 @@ fs_init(void) fs_shared = push_array(arena, FS_Shared, 1); fs_shared->arena = arena; fs_shared->slots_count = 1024; - fs_shared->stripes_count = 64; + fs_shared->stripes_count = os_logical_core_count(); fs_shared->slots = push_array(arena, FS_Slot, fs_shared->slots_count); fs_shared->stripes = push_array(arena, FS_Stripe, fs_shared->stripes_count); for(U64 idx = 0; idx < fs_shared->stripes_count; idx += 1) @@ -30,6 +30,7 @@ fs_init(void) { fs_shared->streamers[idx] = os_launch_thread(fs_streamer_thread__entry_point, (void *)idx, 0); } + fs_shared->detector_thread = os_launch_thread(fs_detector_thread__entry_point, 0, 0); } //////////////////////////////// @@ -188,3 +189,34 @@ fs_streamer_thread__entry_point(void *p) scratch_end(scratch); } } + +//////////////////////////////// +//~ rjf: Change Detector Thread + +internal void +fs_detector_thread__entry_point(void *p) +{ + ThreadNameF("[fs] detector"); + for(;;) + { + U64 slots_per_stripe = fs_shared->slots_count/fs_shared->stripes_count; + for(U64 stripe_idx = 0; stripe_idx < fs_shared->stripes_count; stripe_idx += 1) + { + FS_Stripe *stripe = &fs_shared->stripes[stripe_idx]; + OS_MutexScopeR(stripe->rw_mutex) for(U64 slot_in_stripe_idx = 0; slot_in_stripe_idx < slots_per_stripe; slot_in_stripe_idx += 1) + { + U64 slot_idx = stripe_idx*slots_per_stripe + slot_in_stripe_idx; + FS_Slot *slot = &fs_shared->slots[slot_idx]; + for(FS_Node *n = slot->first; n != 0; n = n->next) + { + FileProperties props = os_properties_from_file_path(n->path); + if(props.modified != n->timestamp && n->last_time_requested_us+100000 < os_now_microseconds()) + { + fs_u2s_enqueue_path(n->path, os_now_microseconds()+100000); + } + } + } + } + os_sleep_milliseconds(100); + } +} diff --git a/src/file_stream/file_stream.h b/src/file_stream/file_stream.h index 4d54a379..8ff10f43 100644 --- a/src/file_stream/file_stream.h +++ b/src/file_stream/file_stream.h @@ -56,6 +56,9 @@ struct FS_Shared // rjf: streamer threads U64 streamer_count; OS_Handle *streamers; + + // rjf: change detector threads + OS_Handle detector_thread; }; //////////////////////////////// @@ -81,4 +84,9 @@ internal String8 fs_u2s_dequeue_path(Arena *arena); internal void fs_streamer_thread__entry_point(void *p); +//////////////////////////////// +//~ rjf: Change Detector Thread + +internal void fs_detector_thread__entry_point(void *p); + #endif // FILE_STREAM_H diff --git a/src/geo_cache/geo_cache.c b/src/geo_cache/geo_cache.c index b871f951..58377510 100644 --- a/src/geo_cache/geo_cache.c +++ b/src/geo_cache/geo_cache.c @@ -11,7 +11,7 @@ geo_init(void) geo_shared = push_array(arena, GEO_Shared, 1); geo_shared->arena = arena; geo_shared->slots_count = 1024; - geo_shared->stripes_count = 64; + geo_shared->stripes_count = Min(geo_shared->slots_count, os_logical_core_count()); geo_shared->slots = push_array(arena, GEO_Slot, geo_shared->slots_count); geo_shared->stripes = push_array(arena, GEO_Stripe, geo_shared->stripes_count); geo_shared->stripes_free_nodes = push_array(arena, GEO_Node *, geo_shared->stripes_count); @@ -22,7 +22,7 @@ geo_init(void) geo_shared->stripes[idx].cv = os_condition_variable_alloc(); } geo_shared->fallback_slots_count = 1024; - geo_shared->fallback_stripes_count = 64; + geo_shared->fallback_stripes_count = Min(geo_shared->fallback_slots_count, os_logical_core_count()); geo_shared->fallback_slots = push_array(arena, GEO_KeyFallbackSlot, geo_shared->fallback_slots_count); geo_shared->fallback_stripes = push_array(arena, GEO_Stripe, geo_shared->fallback_stripes_count); for(U64 idx = 0; idx < geo_shared->fallback_stripes_count; idx += 1) diff --git a/src/hash_store/hash_store.c b/src/hash_store/hash_store.c index c80cd191..5a9570c6 100644 --- a/src/hash_store/hash_store.c +++ b/src/hash_store/hash_store.c @@ -28,7 +28,7 @@ hs_init(void) hs_shared = push_array(arena, HS_Shared, 1); hs_shared->arena = arena; hs_shared->slots_count = 4096; - hs_shared->stripes_count = 64; + hs_shared->stripes_count = Min(hs_shared->slots_count, os_logical_core_count()); hs_shared->slots = push_array(arena, HS_Slot, hs_shared->slots_count); hs_shared->stripes = push_array(arena, HS_Stripe, hs_shared->stripes_count); hs_shared->stripes_free_nodes = push_array(arena, HS_Node *, hs_shared->stripes_count); @@ -40,7 +40,7 @@ hs_init(void) stripe->cv = os_condition_variable_alloc(); } hs_shared->key_slots_count = 4096; - hs_shared->key_stripes_count = 64; + hs_shared->key_stripes_count = Min(hs_shared->key_slots_count, os_logical_core_count()); hs_shared->key_slots = push_array(arena, HS_KeySlot, hs_shared->key_slots_count); hs_shared->key_stripes = push_array(arena, HS_Stripe, hs_shared->key_stripes_count); for(U64 idx = 0; idx < hs_shared->key_stripes_count; idx += 1) diff --git a/src/task_system/task_system.c b/src/task_system/task_system.c index 07a14139..7cc6d1f6 100644 --- a/src/task_system/task_system.c +++ b/src/task_system/task_system.c @@ -27,7 +27,7 @@ ts_init(void) ts_shared = push_array(arena, TS_Shared, 1); ts_shared->arena = arena; ts_shared->artifact_slots_count = 1024; - ts_shared->artifact_stripes_count = 64; + ts_shared->artifact_stripes_count = Min(ts_shared->artifact_slots_count, os_logical_core_count()); ts_shared->artifact_slots = push_array(arena, TS_TaskArtifactSlot, ts_shared->artifact_slots_count); ts_shared->artifact_stripes = push_array(arena, TS_TaskArtifactStripe, ts_shared->artifact_stripes_count); for(U64 idx = 0; idx < ts_shared->artifact_stripes_count; idx += 1) diff --git a/src/text_cache/text_cache.c b/src/text_cache/text_cache.c index 894670c1..33b40521 100644 --- a/src/text_cache/text_cache.c +++ b/src/text_cache/text_cache.c @@ -635,7 +635,7 @@ txt_init(void) txt_shared->arena = arena; txt_shared->slots_count = 1024; txt_shared->slots = push_array(arena, TXT_Slot, txt_shared->slots_count); - txt_shared->stripes_count = 64; + txt_shared->stripes_count = Min(txt_shared->slots_count, os_logical_core_count()); txt_shared->stripes = push_array(arena, TXT_Stripe, txt_shared->stripes_count); txt_shared->stripes_free_nodes = push_array(arena, TXT_Node *, txt_shared->stripes_count); for(U64 idx = 0; idx < txt_shared->stripes_count; idx += 1) @@ -645,7 +645,7 @@ txt_init(void) txt_shared->stripes[idx].cv = os_condition_variable_alloc(); } txt_shared->fallback_slots_count = 256; - txt_shared->fallback_stripes_count = 16; + txt_shared->fallback_stripes_count = Min(txt_shared->fallback_slots_count, os_logical_core_count()); txt_shared->fallback_slots = push_array(arena, TXT_KeyFallbackSlot, txt_shared->fallback_slots_count); txt_shared->fallback_stripes = push_array(arena, TXT_Stripe, txt_shared->fallback_stripes_count); for(U64 idx = 0; idx < txt_shared->fallback_stripes_count; idx += 1) diff --git a/src/texture_cache/texture_cache.c b/src/texture_cache/texture_cache.c index e2897228..9ef652e8 100644 --- a/src/texture_cache/texture_cache.c +++ b/src/texture_cache/texture_cache.c @@ -24,7 +24,7 @@ tex_init(void) tex_shared = push_array(arena, TEX_Shared, 1); tex_shared->arena = arena; tex_shared->slots_count = 1024; - tex_shared->stripes_count = 64; + tex_shared->stripes_count = Min(tex_shared->slots_count, os_logical_core_count()); tex_shared->slots = push_array(arena, TEX_Slot, tex_shared->slots_count); tex_shared->stripes = push_array(arena, TEX_Stripe, tex_shared->stripes_count); tex_shared->stripes_free_nodes = push_array(arena, TEX_Node *, tex_shared->stripes_count); @@ -35,7 +35,7 @@ tex_init(void) tex_shared->stripes[idx].cv = os_condition_variable_alloc(); } tex_shared->fallback_slots_count = 1024; - tex_shared->fallback_stripes_count = 64; + tex_shared->fallback_stripes_count = Min(tex_shared->fallback_slots_count, os_logical_core_count()); tex_shared->fallback_slots = push_array(arena, TEX_KeyFallbackSlot, tex_shared->fallback_slots_count); tex_shared->fallback_stripes = push_array(arena, TEX_Stripe, tex_shared->fallback_stripes_count); for(U64 idx = 0; idx < tex_shared->fallback_stripes_count; idx += 1)