From 0ac6b35a98a86a2f5de30e057d4b5103d0c39f03 Mon Sep 17 00:00:00 2001 From: Nikita Smith Date: Thu, 17 Sep 2026 17:26:25 -0700 Subject: [PATCH] base: add memory placeholders and generic page fault handler * Add windows and Linux demand-mapping backends * Add exclusive file creation, sizing, flushing and replacement APIs --- src/base/base_core.h | 1 + src/base/base_files.h | 4 + src/base/base_inc.c | 3 + src/base/base_inc.h | 1 + src/base/base_memory.h | 14 + src/base/base_shared_memory.h | 1 + src/base/base_system.h | 1 + src/linker/lnk.c | 2 - src/linux/base/linux_base.c | 415 +++++++++++++++++++++++++----- src/linux/base/linux_base.h | 47 +++- src/radbin/radbin_main.c | 2 - src/raddbg/raddbg_main.c | 2 - src/scratch/ryan_scratch.c | 2 - src/scratch/unit_sizes_from_pdb.c | 2 - src/torture/torture_main.c | 2 - src/win32/base/win32_base.c | 234 +++++++++++++++-- src/win32/base/win32_base.h | 12 + src/x64/x64.h | 46 ++++ 18 files changed, 690 insertions(+), 101 deletions(-) diff --git a/src/base/base_core.h b/src/base/base_core.h index 1a21cd7a..580089af 100644 --- a/src/base/base_core.h +++ b/src/base/base_core.h @@ -695,6 +695,7 @@ enum AccessFlag_ShareRead = (1<<4), AccessFlag_ShareWrite = (1<<5), AccessFlag_Inherited = (1<<6), + AccessFlag_CreateNew = (1<<7), // fail if file already exists; never truncates }; //////////////////////////////// diff --git a/src/base/base_files.h b/src/base/base_files.h index b06216c1..c9a98e1d 100644 --- a/src/base/base_files.h +++ b/src/base/base_files.h @@ -89,9 +89,12 @@ internal B32 file_set_times(File file, DateTime time); internal FileProperties properties_from_file(File file); internal FileID id_from_file(File file); internal B32 file_reserve_size(File file, U64 size); +internal B32 file_set_size(File file, U64 size); +internal B32 file_flush(File file); internal B32 delete_file_at_path(String8 path); internal B32 copy_file_path(String8 dst, String8 src); internal B32 move_file_path(String8 dst, String8 src); +internal B32 replace_file_path(String8 dst, String8 src); internal String8 full_path_from_path(Arena *arena, String8 path); internal B32 file_path_exists(String8 path); internal B32 folder_path_exists(String8 path); @@ -101,6 +104,7 @@ internal FileProperties properties_from_file_path(String8 path); internal FileMap file_map_open(AccessFlags flags, File file); internal void file_map_close(FileMap map); internal void * file_map_view_open(FileMap map, AccessFlags flags, Rng1U64 range); +internal void * file_map_view_replace_placeholder(FileMap map, void *ptr, Rng1U64 range); internal void file_map_view_close(FileMap map, void *ptr, Rng1U64 range); //- rjf: directory iteration diff --git a/src/base/base_inc.c b/src/base/base_inc.c index 5a102437..1cff7bf2 100644 --- a/src/base/base_inc.c +++ b/src/base/base_inc.c @@ -30,6 +30,9 @@ #include "base_test.c" #include "base_entry_point.c" +// TODO: move x64 to base layer +#include "x64/x64.c" + #if OS_WINDOWS # include "win32/base/win32_base.c" #elif OS_LINUX diff --git a/src/base/base_inc.h b/src/base/base_inc.h index f83f8858..3aa43a74 100644 --- a/src/base/base_inc.h +++ b/src/base/base_inc.h @@ -32,6 +32,7 @@ #include "base_test.h" #include "base_entry_point.h" +#include "x64/x64.h" #if OS_WINDOWS # include "win32/base/win32_base.h" #elif OS_LINUX diff --git a/src/base/base_memory.h b/src/base/base_memory.h index e73c7afd..6b84fdff 100644 --- a/src/base/base_memory.h +++ b/src/base/base_memory.h @@ -13,6 +13,20 @@ internal B32 commit_memory(void *ptr, U64 size); internal void decommit_memory(void *ptr, U64 size); internal void release_memory(void *ptr, U64 size); +// memory placeholders +internal B32 memory_placeholders_supported(void); +internal void *reserve_memory_placeholders(U64 size, U64 block_size); +internal void release_memory_placeholders(void *ptr, U64 size, U64 block_size); +internal B32 unmap_memory_preserve_placeholder(void *ptr, U64 size); +internal B32 split_memory_placeholder(void *ptr, U64 size, U64 first_size); +internal B32 coalesce_memory_placeholders(void *ptr, U64 size); +union Rng1U64; +internal void prefetch_memory_ranges(U64 count, union Rng1U64 *ranges); + +// generic memory page fault handler (calls may run concurrently) +typedef B32 MemoryReadFaultFunction(void *address, void *user_data); +internal B32 memory_read_fault_handler_set(MemoryReadFaultFunction *func, void *user_data); + //- rjf: large pages internal void *reserve_memory_large(U64 size); internal B32 commit_memory_large(void *ptr, U64 size); diff --git a/src/base/base_shared_memory.h b/src/base/base_shared_memory.h index ec288d86..bd59f0d6 100644 --- a/src/base/base_shared_memory.h +++ b/src/base/base_shared_memory.h @@ -18,5 +18,6 @@ internal SharedMemory shared_memory_open(String8 name); internal void shared_memory_close(SharedMemory handle); internal void * shared_memory_view_open(SharedMemory handle, Rng1U64 range); internal void shared_memory_view_close(SharedMemory handle, void *ptr, Rng1U64 range); +internal void * shared_memory_view_replace_placeholder(SharedMemory handle, void *ptr, Rng1U64 range, AccessFlags flags); #endif // BASE_SHARED_MEMORY_H diff --git a/src/base/base_system.h b/src/base/base_system.h index 2907c881..f28fde04 100644 --- a/src/base/base_system.h +++ b/src/base/base_system.h @@ -11,6 +11,7 @@ struct SystemInfo U64 page_size; U64 large_page_size; U64 allocation_granularity; + U64 physical_memory_size; String8 machine_name; }; diff --git a/src/linker/lnk.c b/src/linker/lnk.c index ad666b0d..423b13bd 100644 --- a/src/linker/lnk.c +++ b/src/linker/lnk.c @@ -13,7 +13,6 @@ // --- Code Base --------------------------------------------------------------- #include "base/base_inc.h" -#include "x64/x64.h" #include "hash_table.h" #include "coff/coff.h" #include "coff/coff_parse.h" @@ -35,7 +34,6 @@ #include "dwarf/x64/dwarf_x64.h" #include "base/base_inc.c" -#include "x64/x64.c" #include "hash_table.c" #include "coff/coff.c" #include "coff/coff_parse.c" diff --git a/src/linux/base/linux_base.c b/src/linux/base/linux_base.c index 3daa94f7..fec7a0da 100644 --- a/src/linux/base/linux_base.c +++ b/src/linux/base/linux_base.c @@ -69,8 +69,10 @@ lnx_file_properties_from_stat(struct stat *s) } internal void -lnx_safe_call_sig_handler(int x) +lnx_safe_call_sig_handler(int sig, siginfo_t *info, void *context) { + if(lnx_dispatch_memory_read_fault(sig, info, context)) { return; } + LNX_SafeCallChain *chain = lnx_safe_call_chain; if(chain != 0 && chain->fail_handler != 0) { @@ -238,8 +240,7 @@ reserve_memory(U64 size) internal B32 commit_memory(void *ptr, U64 size) { - mprotect(ptr, size, PROT_READ|PROT_WRITE); - return 1; + return mprotect(ptr, size, PROT_READ|PROT_WRITE) == 0; } internal void @@ -281,12 +282,29 @@ commit_memory_large(void *ptr, U64 size) internal SharedMemory shared_memory_alloc(U64 size, String8 name) { - Temp scratch = scratch_begin(0, 0); - String8 name_copy = push_str8_copy(scratch.arena, name); - int id = shm_open((char *)name_copy.str, O_RDWR|O_CREAT, 0666); - ftruncate(id, size); - SharedMemory result = {(U64)id}; - scratch_end(scratch); + SharedMemory result = { max_U64 }; + + if(size == 0 || size > max_S64) { return result; } + + int id; + if(name.size == 0) + { + id = memfd_create("rad-shared-memory", MFD_CLOEXEC); + } + else + { + Temp scratch = scratch_begin(0, 0); + String8 name_copy = push_str8_copy(scratch.arena, name); + id = shm_open((char *)name_copy.str, O_RDWR|O_CREAT|O_CLOEXEC, 0666); + scratch_end(scratch); + } + + if(id >= 0) + { + if(ftruncate(id, size) == 0) { result.u64[0] = (U64)id; } + else { close(id); } + } + return result; } @@ -294,9 +312,9 @@ internal SharedMemory shared_memory_open(String8 name) { Temp scratch = scratch_begin(0, 0); - String8 name_copy = push_str8_copy(scratch.arena, name); - int id = shm_open((char *)name_copy.str, O_RDWR, 0); - SharedMemory result = {(U64)id}; + char *name_cstr = (char *)push_str8_copy(scratch.arena, name).str; + int id = shm_open(name_cstr, O_RDWR|O_CLOEXEC, 0); + SharedMemory result = {id >= 0 ? (U64)id : max_U64}; scratch_end(scratch); return result; } @@ -304,15 +322,12 @@ shared_memory_open(String8 name) internal void shared_memory_close(SharedMemory handle) { - if(MemoryIsZeroStruct(&handle)){return;} - int id = (int)handle.u64[0]; - close(id); + close((int)handle.u64[0]); } internal void * shared_memory_view_open(SharedMemory handle, Rng1U64 range) { - if(MemoryIsZeroStruct(&handle)){return 0;} int id = (int)handle.u64[0]; void *base = mmap(0, dim_1u64(range), PROT_READ|PROT_WRITE, MAP_SHARED, id, range.min); if(base == MAP_FAILED) @@ -325,7 +340,6 @@ shared_memory_view_open(SharedMemory handle, Rng1U64 range) internal void shared_memory_view_close(SharedMemory handle, void *ptr, Rng1U64 range) { - if(MemoryIsZeroStruct(&handle)){return;} munmap(ptr, dim_1u64(range)); } @@ -370,7 +384,7 @@ thread_launch(ThreadEntryPointFunctionType *f, void *p) entity->thread.ptr = p; { int pthread_result = pthread_create(&entity->thread.handle, 0, lnx_thread_entry_point, entity); - if(pthread_result == -1) + if(pthread_result != 0) { lnx_entity_release(entity); entity = 0; @@ -760,7 +774,8 @@ safe_call(ThreadEntryPointFunctionType *func, ThreadEntryPointFunctionType *fail // rjf: set up sig handler info struct sigaction new_act = {0}; - new_act.sa_handler = lnx_safe_call_sig_handler; + new_act.sa_sigaction = lnx_safe_call_sig_handler; + new_act.sa_flags = SA_SIGINFO; int signals_to_handle[] = { SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGTRAP, @@ -781,6 +796,7 @@ safe_call(ThreadEntryPointFunctionType *func, ThreadEntryPointFunctionType *fail { sigaction(signals_to_handle[i], &og_act[i], 0); } + lnx_safe_call_chain = chain.next; } //////////////////////////////// @@ -815,6 +831,7 @@ file_open(AccessFlags flags, String8 path) lnx_flags |= O_CREAT; } lnx_flags |= O_CLOEXEC; + if(flags & AccessFlag_CreateNew) { lnx_flags |= O_CREAT|O_EXCL; } int fd = open((char *)path_copy.str, lnx_flags, 0755); File handle = {0}; if(fd != -1) @@ -969,6 +986,265 @@ id_from_file(File file) return id; } +internal B32 +file_set_size(File file, U64 size) +{ + return size <= max_S64 && ftruncate((int)file.u64[0], (off_t)size) == 0; +} + +internal B32 +file_flush(File file) +{ + return fsync((int)file.u64[0]) == 0; +} + +internal B32 +replace_file_path(String8 dst, String8 src) +{ + Temp scratch = scratch_begin(0,0); + char *dst_cstr = (char *)push_str8_copy(scratch.arena, dst).str; + char *src_cstr = (char *)push_str8_copy(scratch.arena, src).str; + B32 result = rename(src_cstr, dst_cstr) == 0; + scratch_end(scratch); + return result; +} + +internal B32 +memory_placeholders_supported(void) +{ +#if ARCH_X64 || ARCH_X86 + return 1; +#else + NotImplemented; + return 0; +#endif +} + +internal void * +reserve_memory_placeholders(U64 size, U64 block_size) +{ + Assert(size > 0); + Assert(size == (size_t)size); + Assert(block_size > 0); + Assert((size % block_size) == 0); + Assert((block_size % get_system_info()->page_size) == 0); + return reserve_memory(size); +} + +internal void +release_memory_placeholders(void *ptr, U64 size, U64 block_size) +{ + if(size && block_size) { release_memory(ptr, size); } +} + +internal B32 +unmap_memory_preserve_placeholder(void *ptr, U64 size) +{ + Assert(ptr != 0); + Assert(size > 0); + Assert(size == (size_t)size); + Assert((IntFromPtr(ptr) % get_system_info()->page_size) == 0); + Assert((size % get_system_info()->page_size) == 0); + return mmap(ptr, size, PROT_NONE, MAP_FIXED|MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) == ptr; +} + +internal void * +shared_memory_view_replace_placeholder(SharedMemory handle, void *ptr, Rng1U64 range, AccessFlags flags) +{ + Assert(range.max > range.min); + Assert(range.max < max_S64); + Assert(dim_1u64(range) == (size_t)dim_1u64(range)); + Assert((range.min % get_system_info()->page_size) == 0); + Assert((range.max % get_system_info()->page_size) == 0); + Assert((IntFromPtr(ptr) % get_system_info()->page_size) == 0); + Assert((flags == AccessFlag_Read) || (flags == (AccessFlag_Read|AccessFlag_Write))); + int protection = flags == AccessFlag_Read ? PROT_READ : PROT_READ|PROT_WRITE; + void *result = mmap(ptr, dim_1u64(range), protection, MAP_FIXED|MAP_SHARED, (int)handle.u64[0], range.min); + return result == MAP_FAILED ? 0 : result; +} + +internal U64 +get_available_commit_memory(void) +{ + long pages = sysconf(_SC_AVPHYS_PAGES); + return pages > 0 ? (U64)pages*get_system_info()->page_size : get_system_info()->physical_memory_size; +} + +internal B32 +split_memory_placeholder(void *ptr, U64 size, U64 first_size) +{ + return first_size > 0 && first_size <= size; +} + +internal B32 +coalesce_memory_placeholders(void *ptr, U64 size) +{ + return 1; +} + +internal void * +file_map_view_replace_placeholder(FileMap map, void *ptr, Rng1U64 range) +{ + void *result = mmap(ptr, dim_1u64(range), PROT_READ, MAP_FIXED|MAP_PRIVATE, (int)map.u64[0], range.min); + return result == MAP_FAILED ? 0 : result; +} + +internal void +prefetch_memory_ranges(U64 count, Rng1U64 *ranges) +{ + U64 page = get_system_info()->page_size; + for EachIndex(i, count) { + U64 min = AlignDownPow2(ranges[i].min, page), max = AlignPow2(ranges[i].max, page); + if (max > min) { madvise(PtrFromInt(min), max-min, MADV_WILLNEED); } + } +} + +internal void +lnx_memory_fault_thread(void *unused) +{ + ThreadNameF("Memory Pager %u", (U32)IntFromPtr(unused)); + for(;;) + { + LNX_MemoryFaultRequest *request = 0; + ssize_t count = LNX_RETRY_ON_EINTR(read(lnx_state.demand_memory.requests[0], &request, sizeof(request))); + + if(count == 0) { break; } + if(count != sizeof(request)) { _exit(1); } + if(request == 0) { break; } + + lnx_in_memory_fault_callback = 1; + B32 handled = lnx_state.demand_memory.fault(request->address, lnx_state.demand_memory.user_data); + lnx_in_memory_fault_callback = 0; + + U32 *completion = &request->result; + ins_atomic_u32_eval_assign(completion, handled ? 2 : 1); + syscall(SYS_futex, completion, FUTEX_WAKE_PRIVATE, 1, 0, 0, 0); + } +} + +internal B32 +lnx_memory_fault_workers_stop(LNX_DemandMemory *memory) +{ + // Explicit stop records work even if a forked child still owns an inherited + // pipe descriptor. EOF-based shutdown could otherwise wait for that child. + LNX_MemoryFaultRequest *stop = 0; + while(memory->stops_sent < memory->worker_count) + { + if(LNX_RETRY_ON_EINTR(write(memory->requests[1], &stop, sizeof(stop))) != sizeof(stop)) { return 0; } + memory->stops_sent += 1; + } + while(memory->workers_joined < memory->worker_count) + { + if(!thread_join(memory->workers[memory->workers_joined], max_U64)) { return 0; } + memory->workers_joined += 1; + } + close(memory->requests[0]); + close(memory->requests[1]); + MemoryZeroStruct(memory); + return 1; +} + +internal B32 +lnx_dispatch_memory_read_fault(int sig, siginfo_t *info, void *context) +{ + LNX_DemandMemory *memory = &lnx_state.demand_memory; + +#if ARCH_X64 || ARCH_X86 + if(sig != SIGSEGV || info == 0 || context == 0 || + (info->si_code != SEGV_ACCERR && info->si_code != SEGV_MAPERR) || + lnx_in_memory_fault_callback || + ins_atomic_u32_eval(&memory->active) == 0) + { + return 0; + } + + ucontext_t *state = context; + if((state->uc_mcontext.gregs[REG_TRAPNO] != X64_Exception_PF) || + (state->uc_mcontext.gregs[REG_ERR] & X64_PageFaultError_Write) || + (state->uc_mcontext.gregs[REG_ERR] & X64_PageFaultError_InstructionFetch) || + memory->owner_pid != getpid()) + { + return 0; + } + + int saved_errno = errno; + LNX_MemoryFaultRequest request = { .address = info->si_addr }; + LNX_MemoryFaultRequest *ptr = &request; + ssize_t count = LNX_RETRY_ON_EINTR(write(memory->requests[1], &ptr, sizeof(ptr))); + if(count != sizeof(ptr)) + { + errno = saved_errno; + return 0; + } + + // wait for the page fault handler to complete + while(!ins_atomic_u32_eval(&request.result)) + { + if(LNX_RETRY_ON_EINTR(syscall(SYS_futex, &request.result, FUTEX_WAIT_PRIVATE, 0, 0, 0, 0)) < 0 && errno != EAGAIN) + { + _exit(1); + } + } + + B32 handled = request.result == 2; + errno = saved_errno; + + return handled; +#else + NotImplemented; + return 0; +#endif +} + +internal B32 +memory_read_fault_handler_set(MemoryReadFaultFunction *func, void *user_data) +{ + LNX_DemandMemory *memory = &lnx_state.demand_memory; + if(func) + { + if(memory_placeholders_supported() == 0 || memory->worker_count || ins_atomic_u32_eval(&memory->active)) + { + return 0; + } + if(pipe2(memory->requests, O_CLOEXEC) != 0) + { + return 0; + } + + memory->fault = func; + memory->user_data = user_data; + memory->owner_pid = getpid(); + // Independent pager workers must remain runnable when all application + // workers are asleep in their SIGSEGV handlers. Fixed-size pointer writes + // fit PIPE_BUF; workers each consume one complete request from the pipe. + U32 worker_count = (U32)Clamp(2, get_system_info()->logical_processor_count, LNX_MEMORY_FAULT_WORKER_LIMIT); + for(U32 i = 0; i < worker_count; i += 1) + { + Thread worker = thread_launch(lnx_memory_fault_thread, PtrFromInt((U64)i)); + if(MemoryIsZeroStruct(&worker)) + { + // Drain and join already-launched workers on partial startup failure. + lnx_memory_fault_workers_stop(memory); + return 0; + } + memory->workers[memory->worker_count++] = worker; + } + + ins_atomic_u32_eval_assign(&memory->active, 1); + } + else if(memory->worker_count) + { + if(memory->owner_pid != getpid() || lnx_in_memory_fault_callback) + { + return 0; + } + + ins_atomic_u32_eval_assign(&memory->active, 0); + return lnx_memory_fault_workers_stop(memory); + } + return 1; +} + internal B32 file_reserve_size(File file, U64 size) { @@ -1523,7 +1799,7 @@ library_open(String8 path) internal void library_close(Library lib) { - void *so = (void *)lib.u64; + void *so = (void *)lib.u64[0]; dlclose(so); } @@ -1531,7 +1807,7 @@ internal VoidProc * library_load_proc(Library lib, String8 name) { Temp scratch = scratch_begin(0, 0); - void *so = (void *)lib.u64; + void *so = (void *)lib.u64[0]; char *name_cstr = (char *)str8_copy(scratch.arena, name).str; VoidProc *proc = (VoidProc *)dlsym(so, name_cstr); scratch_end(scratch); @@ -1544,57 +1820,63 @@ library_load_proc(Library lib, String8 name) internal void lnx_signal_handler(int sig, siginfo_t *info, void *arg) { - local_persist volatile U32 first = 0; - if (ins_atomic_u32_eval_cond_assign(&first, 1, 0) != 0) + // page fault handler + if(lnx_dispatch_memory_read_fault(sig, info, arg)) { return; } + + // crash handler { - for(;;) + local_persist volatile U32 first = 0; + if (ins_atomic_u32_eval_cond_assign(&first, 1, 0) != 0) { - sleep(UINT32_MAX); - } - } - - local_persist void *ips[4096]; - int ips_count = backtrace(ips, ArrayCount(ips)); - - fprintf(stderr, "A fatal signal was received: %s (%d). The process is terminating.\n", strsignal(sig), sig); - fprintf(stderr, "Create a new issue with this report at %s.\n\n", BUILD_ISSUES_LINK_STRING_LITERAL); - fprintf(stderr, "Callstack:\n"); - for EachIndex(i, ips_count) - { - Dl_info info = {0}; - dladdr(ips[i], &info); - - char cmd[2048]; - snprintf(cmd, sizeof(cmd), "llvm-symbolizer --relative-address -f -e %s %lu", info.dli_fname, (unsigned long)ips[i] - (unsigned long)info.dli_fbase); - FILE *f = popen(cmd, "r"); - if(f) - { - char func_name[256], file_name[256]; - if(fgets(func_name, sizeof(func_name), f) && fgets(file_name, sizeof(file_name), f)) + for(;;) { - String8 func = str8_cstring(func_name); - if(func.size > 0) func.size -= 1; - String8 module = str8_skip_last_slash(str8_cstring(info.dli_fname)); - String8 file = str8_skip_last_slash(str8_cstring_capped(file_name, file_name + sizeof(file_name))); - if(file.size > 0) file.size -= 1; - - B32 no_func = str8_match(func, str8_lit("??"), StringMatchFlag_RightSideSloppy); - B32 no_file = str8_match(file, str8_lit("??"), StringMatchFlag_RightSideSloppy); - if(no_func) { func = str8_zero(); } - if(no_file) { file = str8_zero(); } - - fprintf(stderr, "%ld. [0x%016lx] %.*s%s%.*s %.*s\n", i+1, (unsigned long)ips[i], (int)module.size, module.str, (!no_func || !no_file) ? ", " : "", (int)func.size, func.str, (int)file.size, file.str); + sleep(UINT32_MAX); } - pclose(f); } - else + + local_persist void *ips[4096]; + int ips_count = backtrace(ips, ArrayCount(ips)); + + fprintf(stderr, "A fatal signal was received: %s (%d). The process is terminating.\n", strsignal(sig), sig); + fprintf(stderr, "Create a new issue with this report at %s.\n\n", BUILD_ISSUES_LINK_STRING_LITERAL); + fprintf(stderr, "Callstack:\n"); + for EachIndex(i, ips_count) { - fprintf(stderr, "%ld. [0x%016lx] %s\n", i+1, (unsigned long)ips[i], info.dli_fname); + Dl_info info = {0}; + dladdr(ips[i], &info); + + char cmd[2048]; + snprintf(cmd, sizeof(cmd), "llvm-symbolizer --relative-address -f -e %s %lu", info.dli_fname, (unsigned long)ips[i] - (unsigned long)info.dli_fbase); + FILE *f = popen(cmd, "r"); + if(f) + { + char func_name[256], file_name[256]; + if(fgets(func_name, sizeof(func_name), f) && fgets(file_name, sizeof(file_name), f)) + { + String8 func = str8_cstring(func_name); + if(func.size > 0) func.size -= 1; + String8 module = str8_skip_last_slash(str8_cstring(info.dli_fname)); + String8 file = str8_skip_last_slash(str8_cstring_capped(file_name, file_name + sizeof(file_name))); + if(file.size > 0) file.size -= 1; + + B32 no_func = str8_match(func, str8_lit("??"), StringMatchFlag_RightSideSloppy); + B32 no_file = str8_match(file, str8_lit("??"), StringMatchFlag_RightSideSloppy); + if(no_func) { func = str8_zero(); } + if(no_file) { file = str8_zero(); } + + fprintf(stderr, "%ld. [0x%016lx] %.*s%s%.*s %.*s\n", i+1, (unsigned long)ips[i], (int)module.size, module.str, (!no_func || !no_file) ? ", " : "", (int)func.size, func.str, (int)file.size, file.str); + } + pclose(f); + } + else + { + fprintf(stderr, "%ld. [0x%016lx] %s\n", i+1, (unsigned long)ips[i], info.dli_fname); + } } + fprintf(stderr, "\nVersion: %s%s\n\n", BUILD_VERSION_STRING_LITERAL, BUILD_GIT_HASH_STRING_LITERAL_APPEND); + + _exit(1); } - fprintf(stderr, "\nVersion: %s%s\n\n", BUILD_VERSION_STRING_LITERAL, BUILD_GIT_HASH_STRING_LITERAL_APPEND); - - _exit(1); } int @@ -1617,11 +1899,16 @@ main(int argc, char **argv) { //- rjf: get statically-allocated system/process info { + U64 pages = (U64)sysconf(_SC_PHYS_PAGES); + U64 page_size = (U64)sysconf(_SC_PAGESIZE); + U64 commit_size = (U64)sysconf(_SC_AVPHYS_PAGES); + SystemInfo *info = &lnx_state.system_info; info->logical_processor_count = (U32)get_nprocs(); info->page_size = (U64)getpagesize(); info->large_page_size = MB(2); info->allocation_granularity = info->page_size; + info->physical_memory_size = pages * page_size; } { ProcessInfo *info = &lnx_state.process_info; diff --git a/src/linux/base/linux_base.h b/src/linux/base/linux_base.h index 209c6080..87998e59 100644 --- a/src/linux/base/linux_base.h +++ b/src/linux/base/linux_base.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,7 @@ #include #include #include +#include pid_t gettid(void); int pthread_setname_np(pthread_t thread, const char *name); @@ -43,14 +45,14 @@ typedef struct timespec timespec; //////////////////////////////// //~ rjf: Linux Call Interruption Retry Helper -#define LNX_RETRY_ON_EINTR(expr) \ -(__extension__({ \ -__typeof__(expr) __ret; \ -do { \ -__ret = (expr); \ -} while ((__ret == -1) && errno == EINTR); \ -__ret; \ -})) +#define LNX_RETRY_ON_EINTR(expr) \ + (__extension__({ \ + __typeof__(expr) __ret; \ + do { \ + __ret = (expr); \ + } while ((__ret == -1) && errno == EINTR); \ + __ret; \ + })) //////////////////////////////// @@ -113,6 +115,30 @@ struct LNX_Entity }; }; +//////////////////////////////// +//~ On Demand Memory + +#define LNX_MEMORY_FAULT_WORKER_LIMIT 32 + +typedef struct +{ + void *address; + U32 result; +} LNX_MemoryFaultRequest; + +typedef struct +{ + MemoryReadFaultFunction *fault; + void *user_data; + Thread workers[LNX_MEMORY_FAULT_WORKER_LIMIT]; + U32 worker_count; + U32 stops_sent; + U32 workers_joined; + int requests[2]; + U32 active; + pid_t owner_pid; +} LNX_DemandMemory; + //////////////////////////////// //~ rjf: State @@ -127,6 +153,7 @@ struct LNX_State LNX_Entity *entity_free; U64 default_env_count; char **default_env; + LNX_DemandMemory demand_memory; }; //////////////////////////////// @@ -134,6 +161,7 @@ struct LNX_State global LNX_State lnx_state = {0}; thread_static LNX_SafeCallChain *lnx_safe_call_chain = 0; +thread_static B32 lnx_in_memory_fault_callback; //////////////////////////////// //~ rjf: Helpers @@ -143,7 +171,8 @@ internal tm lnx_tm_from_date_time(DateTime dt); internal timespec lnx_timespec_from_date_time(DateTime dt); internal DenseTime lnx_dense_time_from_timespec(timespec in); internal FileProperties lnx_file_properties_from_stat(struct stat *s); -internal void lnx_safe_call_sig_handler(int x); +internal B32 lnx_dispatch_memory_read_fault(int sig, siginfo_t *info, void *context); +internal void lnx_safe_call_sig_handler(int sig, siginfo_t *info, void *context); //////////////////////////////// //~ rjf: Entities diff --git a/src/radbin/radbin_main.c b/src/radbin/radbin_main.c index 3c1fb526..47e351dd 100644 --- a/src/radbin/radbin_main.c +++ b/src/radbin/radbin_main.c @@ -12,7 +12,6 @@ //- rjf: [h] #include "base/base_inc.h" -#include "x64/x64.h" #include "linker/hash_table.h" #include "linker/base_ext/base_bit_array.h" #include "rdi/rdi_local.h" @@ -47,7 +46,6 @@ //- rjf: [c] #include "base/base_inc.c" -#include "x64/x64.c" #include "linker/hash_table.c" #include "linker/base_ext/base_bit_array.c" #include "rdi/rdi_local.c" diff --git a/src/raddbg/raddbg_main.c b/src/raddbg/raddbg_main.c index eb7459c9..b2307cb8 100644 --- a/src/raddbg/raddbg_main.c +++ b/src/raddbg/raddbg_main.c @@ -376,7 +376,6 @@ //- rjf: [h] #include "base/base_inc.h" -#include "x64/x64.h" #include "win32/win32_inc.h" #include "linker/hash_table.h" #include "linker/base_ext/base_bit_array.h" @@ -439,7 +438,6 @@ //- rjf: [c] #include "base/base_inc.c" -#include "x64/x64.c" #include "win32/win32_inc.c" #include "linker/hash_table.c" #include "linker/base_ext/base_bit_array.c" diff --git a/src/scratch/ryan_scratch.c b/src/scratch/ryan_scratch.c index 41a5572d..5509bdac 100644 --- a/src/scratch/ryan_scratch.c +++ b/src/scratch/ryan_scratch.c @@ -12,7 +12,6 @@ //- rjf: [h] #include "base/base_inc.h" -#include "x64/x64.h" #include "artifact_cache/artifact_cache.h" #include "rdi/rdi_local.h" #include "dbg_info/dbg_info.h" @@ -21,7 +20,6 @@ //- rjf: [c] #include "base/base_inc.c" -#include "x64/x64.c" #include "artifact_cache/artifact_cache.c" #include "rdi/rdi_local.c" #include "dbg_info/dbg_info.c" diff --git a/src/scratch/unit_sizes_from_pdb.c b/src/scratch/unit_sizes_from_pdb.c index a2fd7766..4155d33b 100644 --- a/src/scratch/unit_sizes_from_pdb.c +++ b/src/scratch/unit_sizes_from_pdb.c @@ -2,7 +2,6 @@ #define NO_ASYNC 1 #include "base/base_inc.h" -#include "x64/x64.h" #include "linker/hash_table.h" #include "coff/coff.h" #include "coff/coff_parse.h" @@ -25,7 +24,6 @@ #include "dwarf/x64/dwarf_x64.h" #include "base/base_inc.c" -#include "x64/x64.c" #include "linker/hash_table.c" #include "coff/coff.c" #include "coff/coff_parse.c" diff --git a/src/torture/torture_main.c b/src/torture/torture_main.c index 1b3a6915..e7fc5fb1 100644 --- a/src/torture/torture_main.c +++ b/src/torture/torture_main.c @@ -20,7 +20,6 @@ //////////////////////////////// #include "base/base_inc.h" -#include "x64/x64.h" #include "linker/hash_table.h" #include "linker/lf_hash_table.h" #include "linker/base_ext/base_bit_array.h" @@ -104,7 +103,6 @@ #include "torture.h" #include "base/base_inc.c" -#include "x64/x64.c" #include "linker/hash_table.c" #include "linker/lf_hash_table.c" #include "linker/base_ext/base_bit_array.c" diff --git a/src/win32/base/win32_base.c b/src/win32/base/win32_base.c index a67c204c..c2c087a2 100644 --- a/src/win32/base/win32_base.c +++ b/src/win32/base/win32_base.c @@ -6,16 +6,37 @@ // // (We must dynamically link to them, since they can be missing in older SDKs) -typedef HRESULT W32_SetThreadDescription_Type(HANDLE hThread, PCWSTR lpThreadDescription); -global W32_SetThreadDescription_Type *w32_SetThreadDescription_func = 0; -typedef BOOL W32_InitializeSynchronizationBarrier_Type(W32_SYNCHRONIZATION_BARRIER *lpBarrier, LONG lTotalThreads, LONG lSpinCount); -global W32_InitializeSynchronizationBarrier_Type *w32_InitializeSynchronizationBarrier_func = 0; -typedef BOOL W32_DeleteSynchronizationBarrier_Type(W32_SYNCHRONIZATION_BARRIER *lpBarrier); -global W32_DeleteSynchronizationBarrier_Type *w32_DeleteSynchronizationBarrier_func = 0; -typedef BOOL W32_EnterSynchronizationBarrier_Type(W32_SYNCHRONIZATION_BARRIER *lpBarrier, DWORD dwFlags); -global W32_EnterSynchronizationBarrier_Type *w32_EnterSynchronizationBarrier_func = 0; global RIO_EXTENSION_FUNCTION_TABLE w32_rio_functions = {0}; +typedef HRESULT W32_SetThreadDescription_Type(HANDLE hThread, PCWSTR lpThreadDescription); +typedef BOOL W32_InitializeSynchronizationBarrier_Type(W32_SYNCHRONIZATION_BARRIER *lpBarrier, LONG lTotalThreads, LONG lSpinCount); +typedef BOOL W32_DeleteSynchronizationBarrier_Type(W32_SYNCHRONIZATION_BARRIER *lpBarrier); +typedef BOOL W32_EnterSynchronizationBarrier_Type(W32_SYNCHRONIZATION_BARRIER *lpBarrier, DWORD dwFlags); + +typedef struct { void *address; SIZE_T size; } W32_PrefetchRange; +typedef BOOL WINAPI W32_PrefetchVirtualMemory_Type(HANDLE, ULONG_PTR, W32_PrefetchRange *, ULONG); + +#define W32_KERNEL32_EXT_XLIST \ + X(SetThreadDescription) \ + X(InitializeSynchronizationBarrier) \ + X(DeleteSynchronizationBarrier) \ + X(EnterSynchronizationBarrier) \ + X(PrefetchVirtualMemory) + +typedef PVOID W32_VirtualAlloc2_Type(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER *, ULONG); +typedef PVOID W32_MapViewOfFile3_Type(HANDLE, HANDLE, PVOID, ULONG64, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER *, ULONG); +typedef BOOL W32_UnmapViewOfFile2_Type(HANDLE, PVOID, ULONG); + +#define W32_KERNELBASE_EXT_XLIST \ + X(VirtualAlloc2) \ + X(MapViewOfFile3) \ + X(UnmapViewOfFile2) + +#define X(n) global W32_##n##_Type *w32_##n##_func; +W32_KERNEL32_EXT_XLIST +W32_KERNELBASE_EXT_XLIST +#undef X + //////////////////////////////// //~ rjf: File Info Conversion Helpers @@ -304,6 +325,148 @@ release_memory(void *ptr, U64 size) VirtualFree(ptr, 0, MEM_RELEASE); } + +internal B32 +memory_placeholders_supported(void) +{ + return w32_VirtualAlloc2_func && w32_MapViewOfFile3_func && w32_UnmapViewOfFile2_func; +} + +internal void * +reserve_memory_placeholders(U64 size, U64 block_size) +{ + Assert(block_size > 0); + Assert(size > 0); + Assert(size % block_size == 0); + Assert((block_size % get_system_info()->allocation_granularity) == 0); + + // reserve requested placeholder block + U8 *base = w32_VirtualAlloc2_func(GetCurrentProcess(), 0, size, MEM_RESERVE|MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, 0, 0); + if (base == 0) { return 0; } + + // split placeholder block into specified block sizes + for (U64 offset = 0; offset + block_size < size; offset += block_size) + { + if (VirtualFree(base + offset, block_size, MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER) == 0) + { + if (offset == 0 || VirtualFree(base, size, MEM_RELEASE|MEM_COALESCE_PLACEHOLDERS)) + { + VirtualFree(base, 0, MEM_RELEASE); + } + return 0; + } + } + + return base; +} + +internal void +release_memory_placeholders(void *ptr, U64 size, U64 block_size) +{ + Assert(block_size > 0 && size > 0); + if (size > block_size && VirtualFree(ptr, size, MEM_RELEASE|MEM_COALESCE_PLACEHOLDERS) == 0) { return; } + VirtualFree(ptr, 0, MEM_RELEASE); +} + +internal void * +shared_memory_view_replace_placeholder(SharedMemory handle, void *ptr, Rng1U64 range, AccessFlags flags) +{ + Assert(flags == AccessFlag_Read || flags == (AccessFlag_Read|AccessFlag_Write)); + DWORD protection = flags == AccessFlag_Read ? PAGE_READONLY : PAGE_READWRITE; + return w32_MapViewOfFile3_func((HANDLE)handle.u64[0], GetCurrentProcess(), ptr, range.min, dim_1u64(range), MEM_REPLACE_PLACEHOLDER, protection, 0, 0); +} + +internal B32 +unmap_memory_preserve_placeholder(void *ptr, U64 size) +{ + return w32_UnmapViewOfFile2_func(GetCurrentProcess(), ptr, MEM_PRESERVE_PLACEHOLDER); +} + +internal B32 +split_memory_placeholder(void *ptr, U64 size, U64 first_size) +{ + return first_size == size || (first_size && first_size < size && VirtualFree(ptr, first_size, MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER)); +} + +internal B32 +coalesce_memory_placeholders(void *ptr, U64 size) +{ + return VirtualFree(ptr, size, MEM_RELEASE|MEM_COALESCE_PLACEHOLDERS) != 0; +} + +internal void * +file_map_view_replace_placeholder(FileMap map, void *ptr, Rng1U64 range) +{ + return w32_MapViewOfFile3_func((HANDLE)map.u64[0], GetCurrentProcess(), ptr, range.min, dim_1u64(range), MEM_REPLACE_PLACEHOLDER, PAGE_READONLY, 0, 0); +} + +internal void +prefetch_memory_ranges(U64 count, Rng1U64 *ranges) +{ + W32_PrefetchRange batch[256]; + for (U64 first = 0; first < count; first += ArrayCount(batch)) { + U64 n = Min(ArrayCount(batch), count - first); + for EachIndex(i, n) { + batch[i].address = PtrFromInt(ranges[first+i].min); + batch[i].size = dim_1u64(ranges[first+i]); + } + w32_PrefetchVirtualMemory_func(GetCurrentProcess(), n, batch, 0); + } +} + +internal LONG CALLBACK +w32_memory_read_fault(EXCEPTION_POINTERS *exception) +{ + EXCEPTION_RECORD *record = exception->ExceptionRecord; + if (record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && + record->NumberParameters >= 2 && + record->ExceptionInformation[0] == 0 && + w32_state.demand_memory.fault((void *)record->ExceptionInformation[1], w32_state.demand_memory.user_data)) + { + return EXCEPTION_CONTINUE_EXECUTION; + } + return EXCEPTION_CONTINUE_SEARCH; +} + +internal B32 +memory_read_fault_handler_set(MemoryReadFaultFunction *func, void *user_data) +{ + W32_DemandMemory *memory = &w32_state.demand_memory; + + if(func) + { + if (memory->handler == 0) + { + memory->fault = func; + memory->user_data = user_data; + memory->handler = AddVectoredExceptionHandler(1, w32_memory_read_fault); + if(memory->handler == 0) + { + memory->fault = 0; + return 0; + } + } + else { Assert(0 && "handler is already registered"); return 0; } + } + else + { + if (memory->handler) + { + if(RemoveVectoredExceptionHandler(memory->handler) == 0) + { + return 0; + } + memory->handler = 0; + memory->fault = 0; + memory->user_data = 0; + } + else + { Assert(0 && "handler was not registered"); return 0; } + } + + return 1; +} + //- rjf: large pages internal void * @@ -333,7 +496,7 @@ shared_memory_alloc(U64 size, String8 name) PAGE_READWRITE, (U32)((size & 0xffffffff00000000) >> 32), (U32)((size & 0x00000000ffffffff)), - (WCHAR *)name16.str); + name.size ? (WCHAR *)name16.str : 0); SharedMemory result = {(U64)file}; scratch_end(scratch); return result; @@ -778,6 +941,7 @@ file_open(AccessFlags flags, String8 path) if(flags & AccessFlag_ShareWrite) {share_mode |= FILE_SHARE_WRITE|FILE_SHARE_DELETE;} if(flags & AccessFlag_Write) {creation_disposition = CREATE_ALWAYS;} if(flags & AccessFlag_Append) {creation_disposition = OPEN_ALWAYS; access_flags |= FILE_APPEND_DATA; } + if(flags & AccessFlag_CreateNew) {creation_disposition = CREATE_NEW;} if(flags & AccessFlag_Inherited) { security_attributes.bInheritHandle = 1; @@ -983,6 +1147,32 @@ file_reserve_size(File file, U64 size) return is_reserved; } +internal B32 +file_set_size(File file, U64 size) +{ + if (size > max_S64) { return 0; } + FILE_END_OF_FILE_INFO info = {0}; + info.EndOfFile.QuadPart = size; + return SetFileInformationByHandle((HANDLE)file.u64[0], FileEndOfFileInfo, &info, sizeof(info)); +} + +internal B32 +file_flush(File file) +{ + return FlushFileBuffers((HANDLE)file.u64[0]); +} + +internal B32 +replace_file_path(String8 dst, String8 src) +{ + Temp scratch = scratch_begin(0,0); + String16 dst16 = str16_from_8(scratch.arena, dst); + String16 src16 = str16_from_8(scratch.arena, src); + B32 result = MoveFileExW((WCHAR *)src16.str, (WCHAR *)dst16.str, MOVEFILE_REPLACE_EXISTING|MOVEFILE_WRITE_THROUGH); + scratch_end(scratch); + return result; +} + internal B32 delete_file_at_path(String8 path) { @@ -1939,17 +2129,24 @@ w32_entry_point_caller(int argc, WCHAR **wargv) //- rjf: dynamically load windows functions which are not guaranteed // in all SDKs { - HMODULE module = LoadLibraryA("kernel32.dll"); - w32_SetThreadDescription_func = (W32_SetThreadDescription_Type *)GetProcAddress(module, "SetThreadDescription"); - w32_InitializeSynchronizationBarrier_func = (W32_InitializeSynchronizationBarrier_Type *)GetProcAddress(module, "InitializeSynchronizationBarrier"); - w32_DeleteSynchronizationBarrier_func = (W32_DeleteSynchronizationBarrier_Type *)GetProcAddress(module, "DeleteSynchronizationBarrier"); - w32_EnterSynchronizationBarrier_func = (W32_EnterSynchronizationBarrier_Type *)GetProcAddress(module, "EnterSynchronizationBarrier"); + #define X(n) w32_##n##_func = (void*)GetProcAddress(module, Stringify(n)); + HMODULE module; + + module = LoadLibraryA("kernel32.dll"); + W32_KERNEL32_EXT_XLIST + FreeLibrary(module); + + module = LoadLibraryA("kernelbase.dll"); + W32_KERNELBASE_EXT_XLIST + FreeLibrary(module); + if(w32_InitializeSynchronizationBarrier_func == 0) { w32_DeleteSynchronizationBarrier_func = 0; - w32_EnterSynchronizationBarrier_func = 0; + w32_EnterSynchronizationBarrier_func = 0; } - FreeLibrary(module); + + #undef X } //- rjf: try to allow large pages if we can @@ -2000,11 +2197,16 @@ w32_entry_point_caller(int argc, WCHAR **wargv) } } { + + MEMORYSTATUSEX memory = { .dwLength = sizeof(memory) }; + GlobalMemoryStatusEx(&memory); + SystemInfo *info = &w32_state.system_info; info->logical_processor_count = (U64)sysinfo.dwNumberOfProcessors; info->page_size = sysinfo.dwPageSize; info->large_page_size = GetLargePageMinimum(); info->allocation_granularity = sysinfo.dwAllocationGranularity; + info->physical_memory_size = memory.ullTotalPhys; } { ProcessInfo *info = &w32_state.process_info; diff --git a/src/win32/base/win32_base.h b/src/win32/base/win32_base.h index a70812e1..553d7264 100644 --- a/src/win32/base/win32_base.h +++ b/src/win32/base/win32_base.h @@ -88,6 +88,16 @@ struct W32_Entity }; }; +//////////////////////////////// +//~ On Demand Memory + +typedef struct +{ + MemoryReadFaultFunction *fault; + void *user_data; + PVOID handler; +} W32_DemandMemory; + //////////////////////////////// //~ rjf: State @@ -105,6 +115,8 @@ struct W32_State CRITICAL_SECTION entity_mutex; Arena *entity_arena; W32_Entity *entity_free; + + W32_DemandMemory demand_memory; }; //////////////////////////////// diff --git a/src/x64/x64.h b/src/x64/x64.h index db6ec7d2..7d9c1fd1 100644 --- a/src/x64/x64.h +++ b/src/x64/x64.h @@ -209,6 +209,52 @@ typedef enum X64_DebugControlFlags } X64_DebugControlFlags; +//////////////////////////////// +//~ Exception Codes + +typedef enum +{ + X64_Exception_DE = 0, // Divide error: division by zero or quotient overflow + X64_Exception_DB = 1, // Debug exception (single stepping, hardware breakpoint) + X64_Exception_NMI = 2, // Non-maskable interrupt + X64_Exception_BP = 3, // Breakpoint (int3) + X64_Exception_OF = 4, // Overflow + X64_Exception_BR = 5, // Bound range exceeded + X64_Exception_UD = 6, // Invalid opcode + X64_Exception_NM = 7, // Device not available + X64_Exception_DF = 8, // Double fault + X64_Exception_9 = 9, // Coprocessor segment overrun (obsolete) + X64_Exception_TS = 10, // Invalid task-state segment + X64_Exception_NP = 11, // Segment not present + X64_Exception_SS = 12, // Stack-segment fault + X64_Exception_GP = 13, // General-protection fault + X64_Exception_PF = 14, // Page fault + X64_Exception_15 = 15, // Reserved; Linux names it X86_TRAP_SPURIOUS + X64_Exception_MF = 16, // x87 floating-point exception + X64_Exception_AC = 17, // Alignment check + X64_Exception_MC = 18, // Machine check + X64_Exception_XF = 19, // SIMD floating-point exception + X64_Exception_VE = 20, // Virtualization exception + X64_Exception_CP = 21, // Control-protection exception + X64_Exception_Amd_HV = 28, // Hypervisor injection exception + X64_Exception_Amd_VC = 29, // VMM communication exception + X64_Exception_Amd_SX = 30, // Security exception +} X64_ExceptionCode; + +//////////////////////////////// +//~ Page Fault Flags + +typedef enum +{ + X64_PageFaultError_Protection = (1u << 0), + X64_PageFaultError_Write = (1u << 1), + X64_PageFaultError_User = (1u << 2), + X64_PageFaultError_ReservedBit = (1u << 3), + X64_PageFaultError_InstructionFetch = (1u << 4), + X64_PageFaultError_ProtectionKey = (1u << 5), + X64_PageFaultError_ShadowStack = (1u << 6), +} X64_PageFaultError; + //////////////////////////////// //~ rjf: Generated