base: add memory placeholders and generic page fault handler

* Add windows and Linux demand-mapping backends
* Add exclusive file creation, sizing, flushing and replacement APIs
This commit is contained in:
Nikita Smith authored and Ryan Fleury committed 2026-09-21 09:08:07 -07:00
1 parent 4e12148910
commit 0ac6b35a98
18 files changed
+690 -101

No files matched your search

+1
View File
@@ -695,6 +695,7 @@ enum
AccessFlag_ShareRead = (1<<4),
AccessFlag_ShareWrite = (1<<5),
AccessFlag_Inherited = (1<<6),
AccessFlag_CreateNew = (1<<7), // fail if file already exists; never truncates
};
////////////////////////////////
+4
View File
@@ -89,9 +89,12 @@ internal B32 file_set_times(File file, DateTime time);
internal FileProperties properties_from_file(File file);
internal FileID id_from_file(File file);
internal B32 file_reserve_size(File file, U64 size);
internal B32 file_set_size(File file, U64 size);
internal B32 file_flush(File file);
internal B32 delete_file_at_path(String8 path);
internal B32 copy_file_path(String8 dst, String8 src);
internal B32 move_file_path(String8 dst, String8 src);
internal B32 replace_file_path(String8 dst, String8 src);
internal String8 full_path_from_path(Arena *arena, String8 path);
internal B32 file_path_exists(String8 path);
internal B32 folder_path_exists(String8 path);
@@ -101,6 +104,7 @@ internal FileProperties properties_from_file_path(String8 path);
internal FileMap file_map_open(AccessFlags flags, File file);
internal void file_map_close(FileMap map);
internal void * file_map_view_open(FileMap map, AccessFlags flags, Rng1U64 range);
internal void * file_map_view_replace_placeholder(FileMap map, void *ptr, Rng1U64 range);
internal void file_map_view_close(FileMap map, void *ptr, Rng1U64 range);
//- rjf: directory iteration
+3
View File
@@ -30,6 +30,9 @@
#include "base_test.c"
#include "base_entry_point.c"
// TODO: move x64 to base layer
#include "x64/x64.c"
#if OS_WINDOWS
# include "win32/base/win32_base.c"
#elif OS_LINUX
+1
View File
@@ -32,6 +32,7 @@
#include "base_test.h"
#include "base_entry_point.h"
#include "x64/x64.h"
#if OS_WINDOWS
# include "win32/base/win32_base.h"
#elif OS_LINUX
+14
View File
@@ -13,6 +13,20 @@ internal B32 commit_memory(void *ptr, U64 size);
internal void decommit_memory(void *ptr, U64 size);
internal void release_memory(void *ptr, U64 size);
// memory placeholders
internal B32 memory_placeholders_supported(void);
internal void *reserve_memory_placeholders(U64 size, U64 block_size);
internal void release_memory_placeholders(void *ptr, U64 size, U64 block_size);
internal B32 unmap_memory_preserve_placeholder(void *ptr, U64 size);
internal B32 split_memory_placeholder(void *ptr, U64 size, U64 first_size);
internal B32 coalesce_memory_placeholders(void *ptr, U64 size);
union Rng1U64;
internal void prefetch_memory_ranges(U64 count, union Rng1U64 *ranges);
// generic memory page fault handler (calls may run concurrently)
typedef B32 MemoryReadFaultFunction(void *address, void *user_data);
internal B32 memory_read_fault_handler_set(MemoryReadFaultFunction *func, void *user_data);
//- rjf: large pages
internal void *reserve_memory_large(U64 size);
internal B32 commit_memory_large(void *ptr, U64 size);
+1
View File
@@ -18,5 +18,6 @@ internal SharedMemory shared_memory_open(String8 name);
internal void shared_memory_close(SharedMemory handle);
internal void * shared_memory_view_open(SharedMemory handle, Rng1U64 range);
internal void shared_memory_view_close(SharedMemory handle, void *ptr, Rng1U64 range);
internal void * shared_memory_view_replace_placeholder(SharedMemory handle, void *ptr, Rng1U64 range, AccessFlags flags);
#endif // BASE_SHARED_MEMORY_H
+1
View File
@@ -11,6 +11,7 @@ struct SystemInfo
U64 page_size;
U64 large_page_size;
U64 allocation_granularity;
U64 physical_memory_size;
String8 machine_name;
};
-2
View File
@@ -13,7 +13,6 @@
// --- Code Base ---------------------------------------------------------------
#include "base/base_inc.h"
#include "x64/x64.h"
#include "hash_table.h"
#include "coff/coff.h"
#include "coff/coff_parse.h"
@@ -35,7 +34,6 @@
#include "dwarf/x64/dwarf_x64.h"
#include "base/base_inc.c"
#include "x64/x64.c"
#include "hash_table.c"
#include "coff/coff.c"
#include "coff/coff_parse.c"
+351 -64
View File
@@ -69,8 +69,10 @@ lnx_file_properties_from_stat(struct stat *s)
}
internal void
lnx_safe_call_sig_handler(int x)
lnx_safe_call_sig_handler(int sig, siginfo_t *info, void *context)
{
if(lnx_dispatch_memory_read_fault(sig, info, context)) { return; }
LNX_SafeCallChain *chain = lnx_safe_call_chain;
if(chain != 0 && chain->fail_handler != 0)
{
@@ -238,8 +240,7 @@ reserve_memory(U64 size)
internal B32
commit_memory(void *ptr, U64 size)
{
mprotect(ptr, size, PROT_READ|PROT_WRITE);
return 1;
return mprotect(ptr, size, PROT_READ|PROT_WRITE) == 0;
}
internal void
@@ -281,12 +282,29 @@ commit_memory_large(void *ptr, U64 size)
internal SharedMemory
shared_memory_alloc(U64 size, String8 name)
{
Temp scratch = scratch_begin(0, 0);
String8 name_copy = push_str8_copy(scratch.arena, name);
int id = shm_open((char *)name_copy.str, O_RDWR|O_CREAT, 0666);
ftruncate(id, size);
SharedMemory result = {(U64)id};
scratch_end(scratch);
SharedMemory result = { max_U64 };
if(size == 0 || size > max_S64) { return result; }
int id;
if(name.size == 0)
{
id = memfd_create("rad-shared-memory", MFD_CLOEXEC);
}
else
{
Temp scratch = scratch_begin(0, 0);
String8 name_copy = push_str8_copy(scratch.arena, name);
id = shm_open((char *)name_copy.str, O_RDWR|O_CREAT|O_CLOEXEC, 0666);
scratch_end(scratch);
}
if(id >= 0)
{
if(ftruncate(id, size) == 0) { result.u64[0] = (U64)id; }
else { close(id); }
}
return result;
}
@@ -294,9 +312,9 @@ internal SharedMemory
shared_memory_open(String8 name)
{
Temp scratch = scratch_begin(0, 0);
String8 name_copy = push_str8_copy(scratch.arena, name);
int id = shm_open((char *)name_copy.str, O_RDWR, 0);
SharedMemory result = {(U64)id};
char *name_cstr = (char *)push_str8_copy(scratch.arena, name).str;
int id = shm_open(name_cstr, O_RDWR|O_CLOEXEC, 0);
SharedMemory result = {id >= 0 ? (U64)id : max_U64};
scratch_end(scratch);
return result;
}
@@ -304,15 +322,12 @@ shared_memory_open(String8 name)
internal void
shared_memory_close(SharedMemory handle)
{
if(MemoryIsZeroStruct(&handle)){return;}
int id = (int)handle.u64[0];
close(id);
close((int)handle.u64[0]);
}
internal void *
shared_memory_view_open(SharedMemory handle, Rng1U64 range)
{
if(MemoryIsZeroStruct(&handle)){return 0;}
int id = (int)handle.u64[0];
void *base = mmap(0, dim_1u64(range), PROT_READ|PROT_WRITE, MAP_SHARED, id, range.min);
if(base == MAP_FAILED)
@@ -325,7 +340,6 @@ shared_memory_view_open(SharedMemory handle, Rng1U64 range)
internal void
shared_memory_view_close(SharedMemory handle, void *ptr, Rng1U64 range)
{
if(MemoryIsZeroStruct(&handle)){return;}
munmap(ptr, dim_1u64(range));
}
@@ -370,7 +384,7 @@ thread_launch(ThreadEntryPointFunctionType *f, void *p)
entity->thread.ptr = p;
{
int pthread_result = pthread_create(&entity->thread.handle, 0, lnx_thread_entry_point, entity);
if(pthread_result == -1)
if(pthread_result != 0)
{
lnx_entity_release(entity);
entity = 0;
@@ -760,7 +774,8 @@ safe_call(ThreadEntryPointFunctionType *func, ThreadEntryPointFunctionType *fail
// rjf: set up sig handler info
struct sigaction new_act = {0};
new_act.sa_handler = lnx_safe_call_sig_handler;
new_act.sa_sigaction = lnx_safe_call_sig_handler;
new_act.sa_flags = SA_SIGINFO;
int signals_to_handle[] =
{
SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGTRAP,
@@ -781,6 +796,7 @@ safe_call(ThreadEntryPointFunctionType *func, ThreadEntryPointFunctionType *fail
{
sigaction(signals_to_handle[i], &og_act[i], 0);
}
lnx_safe_call_chain = chain.next;
}
////////////////////////////////
@@ -815,6 +831,7 @@ file_open(AccessFlags flags, String8 path)
lnx_flags |= O_CREAT;
}
lnx_flags |= O_CLOEXEC;
if(flags & AccessFlag_CreateNew) { lnx_flags |= O_CREAT|O_EXCL; }
int fd = open((char *)path_copy.str, lnx_flags, 0755);
File handle = {0};
if(fd != -1)
@@ -969,6 +986,265 @@ id_from_file(File file)
return id;
}
internal B32
file_set_size(File file, U64 size)
{
return size <= max_S64 && ftruncate((int)file.u64[0], (off_t)size) == 0;
}
internal B32
file_flush(File file)
{
return fsync((int)file.u64[0]) == 0;
}
internal B32
replace_file_path(String8 dst, String8 src)
{
Temp scratch = scratch_begin(0,0);
char *dst_cstr = (char *)push_str8_copy(scratch.arena, dst).str;
char *src_cstr = (char *)push_str8_copy(scratch.arena, src).str;
B32 result = rename(src_cstr, dst_cstr) == 0;
scratch_end(scratch);
return result;
}
internal B32
memory_placeholders_supported(void)
{
#if ARCH_X64 || ARCH_X86
return 1;
#else
NotImplemented;
return 0;
#endif
}
internal void *
reserve_memory_placeholders(U64 size, U64 block_size)
{
Assert(size > 0);
Assert(size == (size_t)size);
Assert(block_size > 0);
Assert((size % block_size) == 0);
Assert((block_size % get_system_info()->page_size) == 0);
return reserve_memory(size);
}
internal void
release_memory_placeholders(void *ptr, U64 size, U64 block_size)
{
if(size && block_size) { release_memory(ptr, size); }
}
internal B32
unmap_memory_preserve_placeholder(void *ptr, U64 size)
{
Assert(ptr != 0);
Assert(size > 0);
Assert(size == (size_t)size);
Assert((IntFromPtr(ptr) % get_system_info()->page_size) == 0);
Assert((size % get_system_info()->page_size) == 0);
return mmap(ptr, size, PROT_NONE, MAP_FIXED|MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) == ptr;
}
internal void *
shared_memory_view_replace_placeholder(SharedMemory handle, void *ptr, Rng1U64 range, AccessFlags flags)
{
Assert(range.max > range.min);
Assert(range.max < max_S64);
Assert(dim_1u64(range) == (size_t)dim_1u64(range));
Assert((range.min % get_system_info()->page_size) == 0);
Assert((range.max % get_system_info()->page_size) == 0);
Assert((IntFromPtr(ptr) % get_system_info()->page_size) == 0);
Assert((flags == AccessFlag_Read) || (flags == (AccessFlag_Read|AccessFlag_Write)));
int protection = flags == AccessFlag_Read ? PROT_READ : PROT_READ|PROT_WRITE;
void *result = mmap(ptr, dim_1u64(range), protection, MAP_FIXED|MAP_SHARED, (int)handle.u64[0], range.min);
return result == MAP_FAILED ? 0 : result;
}
internal U64
get_available_commit_memory(void)
{
long pages = sysconf(_SC_AVPHYS_PAGES);
return pages > 0 ? (U64)pages*get_system_info()->page_size : get_system_info()->physical_memory_size;
}
internal B32
split_memory_placeholder(void *ptr, U64 size, U64 first_size)
{
return first_size > 0 && first_size <= size;
}
internal B32
coalesce_memory_placeholders(void *ptr, U64 size)
{
return 1;
}
internal void *
file_map_view_replace_placeholder(FileMap map, void *ptr, Rng1U64 range)
{
void *result = mmap(ptr, dim_1u64(range), PROT_READ, MAP_FIXED|MAP_PRIVATE, (int)map.u64[0], range.min);
return result == MAP_FAILED ? 0 : result;
}
internal void
prefetch_memory_ranges(U64 count, Rng1U64 *ranges)
{
U64 page = get_system_info()->page_size;
for EachIndex(i, count) {
U64 min = AlignDownPow2(ranges[i].min, page), max = AlignPow2(ranges[i].max, page);
if (max > min) { madvise(PtrFromInt(min), max-min, MADV_WILLNEED); }
}
}
internal void
lnx_memory_fault_thread(void *unused)
{
ThreadNameF("Memory Pager %u", (U32)IntFromPtr(unused));
for(;;)
{
LNX_MemoryFaultRequest *request = 0;
ssize_t count = LNX_RETRY_ON_EINTR(read(lnx_state.demand_memory.requests[0], &request, sizeof(request)));
if(count == 0) { break; }
if(count != sizeof(request)) { _exit(1); }
if(request == 0) { break; }
lnx_in_memory_fault_callback = 1;
B32 handled = lnx_state.demand_memory.fault(request->address, lnx_state.demand_memory.user_data);
lnx_in_memory_fault_callback = 0;
U32 *completion = &request->result;
ins_atomic_u32_eval_assign(completion, handled ? 2 : 1);
syscall(SYS_futex, completion, FUTEX_WAKE_PRIVATE, 1, 0, 0, 0);
}
}
internal B32
lnx_memory_fault_workers_stop(LNX_DemandMemory *memory)
{
// Explicit stop records work even if a forked child still owns an inherited
// pipe descriptor. EOF-based shutdown could otherwise wait for that child.
LNX_MemoryFaultRequest *stop = 0;
while(memory->stops_sent < memory->worker_count)
{
if(LNX_RETRY_ON_EINTR(write(memory->requests[1], &stop, sizeof(stop))) != sizeof(stop)) { return 0; }
memory->stops_sent += 1;
}
while(memory->workers_joined < memory->worker_count)
{
if(!thread_join(memory->workers[memory->workers_joined], max_U64)) { return 0; }
memory->workers_joined += 1;
}
close(memory->requests[0]);
close(memory->requests[1]);
MemoryZeroStruct(memory);
return 1;
}
internal B32
lnx_dispatch_memory_read_fault(int sig, siginfo_t *info, void *context)
{
LNX_DemandMemory *memory = &lnx_state.demand_memory;
#if ARCH_X64 || ARCH_X86
if(sig != SIGSEGV || info == 0 || context == 0 ||
(info->si_code != SEGV_ACCERR && info->si_code != SEGV_MAPERR) ||
lnx_in_memory_fault_callback ||
ins_atomic_u32_eval(&memory->active) == 0)
{
return 0;
}
ucontext_t *state = context;
if((state->uc_mcontext.gregs[REG_TRAPNO] != X64_Exception_PF) ||
(state->uc_mcontext.gregs[REG_ERR] & X64_PageFaultError_Write) ||
(state->uc_mcontext.gregs[REG_ERR] & X64_PageFaultError_InstructionFetch) ||
memory->owner_pid != getpid())
{
return 0;
}
int saved_errno = errno;
LNX_MemoryFaultRequest request = { .address = info->si_addr };
LNX_MemoryFaultRequest *ptr = &request;
ssize_t count = LNX_RETRY_ON_EINTR(write(memory->requests[1], &ptr, sizeof(ptr)));
if(count != sizeof(ptr))
{
errno = saved_errno;
return 0;
}
// wait for the page fault handler to complete
while(!ins_atomic_u32_eval(&request.result))
{
if(LNX_RETRY_ON_EINTR(syscall(SYS_futex, &request.result, FUTEX_WAIT_PRIVATE, 0, 0, 0, 0)) < 0 && errno != EAGAIN)
{
_exit(1);
}
}
B32 handled = request.result == 2;
errno = saved_errno;
return handled;
#else
NotImplemented;
return 0;
#endif
}
internal B32
memory_read_fault_handler_set(MemoryReadFaultFunction *func, void *user_data)
{
LNX_DemandMemory *memory = &lnx_state.demand_memory;
if(func)
{
if(memory_placeholders_supported() == 0 || memory->worker_count || ins_atomic_u32_eval(&memory->active))
{
return 0;
}
if(pipe2(memory->requests, O_CLOEXEC) != 0)
{
return 0;
}
memory->fault = func;
memory->user_data = user_data;
memory->owner_pid = getpid();
// Independent pager workers must remain runnable when all application
// workers are asleep in their SIGSEGV handlers. Fixed-size pointer writes
// fit PIPE_BUF; workers each consume one complete request from the pipe.
U32 worker_count = (U32)Clamp(2, get_system_info()->logical_processor_count, LNX_MEMORY_FAULT_WORKER_LIMIT);
for(U32 i = 0; i < worker_count; i += 1)
{
Thread worker = thread_launch(lnx_memory_fault_thread, PtrFromInt((U64)i));
if(MemoryIsZeroStruct(&worker))
{
// Drain and join already-launched workers on partial startup failure.
lnx_memory_fault_workers_stop(memory);
return 0;
}
memory->workers[memory->worker_count++] = worker;
}
ins_atomic_u32_eval_assign(&memory->active, 1);
}
else if(memory->worker_count)
{
if(memory->owner_pid != getpid() || lnx_in_memory_fault_callback)
{
return 0;
}
ins_atomic_u32_eval_assign(&memory->active, 0);
return lnx_memory_fault_workers_stop(memory);
}
return 1;
}
internal B32
file_reserve_size(File file, U64 size)
{
@@ -1523,7 +1799,7 @@ library_open(String8 path)
internal void
library_close(Library lib)
{
void *so = (void *)lib.u64;
void *so = (void *)lib.u64[0];
dlclose(so);
}
@@ -1531,7 +1807,7 @@ internal VoidProc *
library_load_proc(Library lib, String8 name)
{
Temp scratch = scratch_begin(0, 0);
void *so = (void *)lib.u64;
void *so = (void *)lib.u64[0];
char *name_cstr = (char *)str8_copy(scratch.arena, name).str;
VoidProc *proc = (VoidProc *)dlsym(so, name_cstr);
scratch_end(scratch);
@@ -1544,57 +1820,63 @@ library_load_proc(Library lib, String8 name)
internal void
lnx_signal_handler(int sig, siginfo_t *info, void *arg)
{
local_persist volatile U32 first = 0;
if (ins_atomic_u32_eval_cond_assign(&first, 1, 0) != 0)
// page fault handler
if(lnx_dispatch_memory_read_fault(sig, info, arg)) { return; }
// crash handler
{
for(;;)
local_persist volatile U32 first = 0;
if (ins_atomic_u32_eval_cond_assign(&first, 1, 0) != 0)
{
sleep(UINT32_MAX);
}
}
local_persist void *ips[4096];
int ips_count = backtrace(ips, ArrayCount(ips));
fprintf(stderr, "A fatal signal was received: %s (%d). The process is terminating.\n", strsignal(sig), sig);
fprintf(stderr, "Create a new issue with this report at %s.\n\n", BUILD_ISSUES_LINK_STRING_LITERAL);
fprintf(stderr, "Callstack:\n");
for EachIndex(i, ips_count)
{
Dl_info info = {0};
dladdr(ips[i], &info);
char cmd[2048];
snprintf(cmd, sizeof(cmd), "llvm-symbolizer --relative-address -f -e %s %lu", info.dli_fname, (unsigned long)ips[i] - (unsigned long)info.dli_fbase);
FILE *f = popen(cmd, "r");
if(f)
{
char func_name[256], file_name[256];
if(fgets(func_name, sizeof(func_name), f) && fgets(file_name, sizeof(file_name), f))
for(;;)
{
String8 func = str8_cstring(func_name);
if(func.size > 0) func.size -= 1;
String8 module = str8_skip_last_slash(str8_cstring(info.dli_fname));
String8 file = str8_skip_last_slash(str8_cstring_capped(file_name, file_name + sizeof(file_name)));
if(file.size > 0) file.size -= 1;
B32 no_func = str8_match(func, str8_lit("??"), StringMatchFlag_RightSideSloppy);
B32 no_file = str8_match(file, str8_lit("??"), StringMatchFlag_RightSideSloppy);
if(no_func) { func = str8_zero(); }
if(no_file) { file = str8_zero(); }
fprintf(stderr, "%ld. [0x%016lx] %.*s%s%.*s %.*s\n", i+1, (unsigned long)ips[i], (int)module.size, module.str, (!no_func || !no_file) ? ", " : "", (int)func.size, func.str, (int)file.size, file.str);
sleep(UINT32_MAX);
}
pclose(f);
}
else
local_persist void *ips[4096];
int ips_count = backtrace(ips, ArrayCount(ips));
fprintf(stderr, "A fatal signal was received: %s (%d). The process is terminating.\n", strsignal(sig), sig);
fprintf(stderr, "Create a new issue with this report at %s.\n\n", BUILD_ISSUES_LINK_STRING_LITERAL);
fprintf(stderr, "Callstack:\n");
for EachIndex(i, ips_count)
{
fprintf(stderr, "%ld. [0x%016lx] %s\n", i+1, (unsigned long)ips[i], info.dli_fname);
Dl_info info = {0};
dladdr(ips[i], &info);
char cmd[2048];
snprintf(cmd, sizeof(cmd), "llvm-symbolizer --relative-address -f -e %s %lu", info.dli_fname, (unsigned long)ips[i] - (unsigned long)info.dli_fbase);
FILE *f = popen(cmd, "r");
if(f)
{
char func_name[256], file_name[256];
if(fgets(func_name, sizeof(func_name), f) && fgets(file_name, sizeof(file_name), f))
{
String8 func = str8_cstring(func_name);
if(func.size > 0) func.size -= 1;
String8 module = str8_skip_last_slash(str8_cstring(info.dli_fname));
String8 file = str8_skip_last_slash(str8_cstring_capped(file_name, file_name + sizeof(file_name)));
if(file.size > 0) file.size -= 1;
B32 no_func = str8_match(func, str8_lit("??"), StringMatchFlag_RightSideSloppy);
B32 no_file = str8_match(file, str8_lit("??"), StringMatchFlag_RightSideSloppy);
if(no_func) { func = str8_zero(); }
if(no_file) { file = str8_zero(); }
fprintf(stderr, "%ld. [0x%016lx] %.*s%s%.*s %.*s\n", i+1, (unsigned long)ips[i], (int)module.size, module.str, (!no_func || !no_file) ? ", " : "", (int)func.size, func.str, (int)file.size, file.str);
}
pclose(f);
}
else
{
fprintf(stderr, "%ld. [0x%016lx] %s\n", i+1, (unsigned long)ips[i], info.dli_fname);
}
}
fprintf(stderr, "\nVersion: %s%s\n\n", BUILD_VERSION_STRING_LITERAL, BUILD_GIT_HASH_STRING_LITERAL_APPEND);
_exit(1);
}
fprintf(stderr, "\nVersion: %s%s\n\n", BUILD_VERSION_STRING_LITERAL, BUILD_GIT_HASH_STRING_LITERAL_APPEND);
_exit(1);
}
int
@@ -1617,11 +1899,16 @@ main(int argc, char **argv)
{
//- rjf: get statically-allocated system/process info
{
U64 pages = (U64)sysconf(_SC_PHYS_PAGES);
U64 page_size = (U64)sysconf(_SC_PAGESIZE);
U64 commit_size = (U64)sysconf(_SC_AVPHYS_PAGES);
SystemInfo *info = &lnx_state.system_info;
info->logical_processor_count = (U32)get_nprocs();
info->page_size = (U64)getpagesize();
info->large_page_size = MB(2);
info->allocation_granularity = info->page_size;
info->physical_memory_size = pages * page_size;
}
{
ProcessInfo *info = &lnx_state.process_info;
+38 -9
View File
@@ -15,6 +15,7 @@
#include <fcntl.h>
#include <features.h>
#include <linux/limits.h>
#include <linux/futex.h>
#include <poll.h>
#include <pthread.h>
#include <semaphore.h>
@@ -32,6 +33,7 @@
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>
#include <ucontext.h>
pid_t gettid(void);
int pthread_setname_np(pthread_t thread, const char *name);
@@ -43,14 +45,14 @@ typedef struct timespec timespec;
////////////////////////////////
//~ rjf: Linux Call Interruption Retry Helper
#define LNX_RETRY_ON_EINTR(expr) \
(__extension__({ \
__typeof__(expr) __ret; \
do { \
__ret = (expr); \
} while ((__ret == -1) && errno == EINTR); \
__ret; \
}))
#define LNX_RETRY_ON_EINTR(expr) \
(__extension__({ \
__typeof__(expr) __ret; \
do { \
__ret = (expr); \
} while ((__ret == -1) && errno == EINTR); \
__ret; \
}))
////////////////////////////////
@@ -113,6 +115,30 @@ struct LNX_Entity
};
};
////////////////////////////////
//~ On Demand Memory
#define LNX_MEMORY_FAULT_WORKER_LIMIT 32
typedef struct
{
void *address;
U32 result;
} LNX_MemoryFaultRequest;
typedef struct
{
MemoryReadFaultFunction *fault;
void *user_data;
Thread workers[LNX_MEMORY_FAULT_WORKER_LIMIT];
U32 worker_count;
U32 stops_sent;
U32 workers_joined;
int requests[2];
U32 active;
pid_t owner_pid;
} LNX_DemandMemory;
////////////////////////////////
//~ rjf: State
@@ -127,6 +153,7 @@ struct LNX_State
LNX_Entity *entity_free;
U64 default_env_count;
char **default_env;
LNX_DemandMemory demand_memory;
};
////////////////////////////////
@@ -134,6 +161,7 @@ struct LNX_State
global LNX_State lnx_state = {0};
thread_static LNX_SafeCallChain *lnx_safe_call_chain = 0;
thread_static B32 lnx_in_memory_fault_callback;
////////////////////////////////
//~ rjf: Helpers
@@ -143,7 +171,8 @@ internal tm lnx_tm_from_date_time(DateTime dt);
internal timespec lnx_timespec_from_date_time(DateTime dt);
internal DenseTime lnx_dense_time_from_timespec(timespec in);
internal FileProperties lnx_file_properties_from_stat(struct stat *s);
internal void lnx_safe_call_sig_handler(int x);
internal B32 lnx_dispatch_memory_read_fault(int sig, siginfo_t *info, void *context);
internal void lnx_safe_call_sig_handler(int sig, siginfo_t *info, void *context);
////////////////////////////////
//~ rjf: Entities
-2
View File
@@ -12,7 +12,6 @@
//- rjf: [h]
#include "base/base_inc.h"
#include "x64/x64.h"
#include "linker/hash_table.h"
#include "linker/base_ext/base_bit_array.h"
#include "rdi/rdi_local.h"
@@ -47,7 +46,6 @@
//- rjf: [c]
#include "base/base_inc.c"
#include "x64/x64.c"
#include "linker/hash_table.c"
#include "linker/base_ext/base_bit_array.c"
#include "rdi/rdi_local.c"
-2
View File
@@ -376,7 +376,6 @@
//- rjf: [h]
#include "base/base_inc.h"
#include "x64/x64.h"
#include "win32/win32_inc.h"
#include "linker/hash_table.h"
#include "linker/base_ext/base_bit_array.h"
@@ -439,7 +438,6 @@
//- rjf: [c]
#include "base/base_inc.c"
#include "x64/x64.c"
#include "win32/win32_inc.c"
#include "linker/hash_table.c"
#include "linker/base_ext/base_bit_array.c"
-2
View File
@@ -12,7 +12,6 @@
//- rjf: [h]
#include "base/base_inc.h"
#include "x64/x64.h"
#include "artifact_cache/artifact_cache.h"
#include "rdi/rdi_local.h"
#include "dbg_info/dbg_info.h"
@@ -21,7 +20,6 @@
//- rjf: [c]
#include "base/base_inc.c"
#include "x64/x64.c"
#include "artifact_cache/artifact_cache.c"
#include "rdi/rdi_local.c"
#include "dbg_info/dbg_info.c"
-2
View File
@@ -2,7 +2,6 @@
#define NO_ASYNC 1
#include "base/base_inc.h"
#include "x64/x64.h"
#include "linker/hash_table.h"
#include "coff/coff.h"
#include "coff/coff_parse.h"
@@ -25,7 +24,6 @@
#include "dwarf/x64/dwarf_x64.h"
#include "base/base_inc.c"
#include "x64/x64.c"
#include "linker/hash_table.c"
#include "coff/coff.c"
#include "coff/coff_parse.c"
-2
View File
@@ -20,7 +20,6 @@
////////////////////////////////
#include "base/base_inc.h"
#include "x64/x64.h"
#include "linker/hash_table.h"
#include "linker/lf_hash_table.h"
#include "linker/base_ext/base_bit_array.h"
@@ -104,7 +103,6 @@
#include "torture.h"
#include "base/base_inc.c"
#include "x64/x64.c"
#include "linker/hash_table.c"
#include "linker/lf_hash_table.c"
#include "linker/base_ext/base_bit_array.c"
+218 -16
View File
@@ -6,16 +6,37 @@
//
// (We must dynamically link to them, since they can be missing in older SDKs)
typedef HRESULT W32_SetThreadDescription_Type(HANDLE hThread, PCWSTR lpThreadDescription);
global W32_SetThreadDescription_Type *w32_SetThreadDescription_func = 0;
typedef BOOL W32_InitializeSynchronizationBarrier_Type(W32_SYNCHRONIZATION_BARRIER *lpBarrier, LONG lTotalThreads, LONG lSpinCount);
global W32_InitializeSynchronizationBarrier_Type *w32_InitializeSynchronizationBarrier_func = 0;
typedef BOOL W32_DeleteSynchronizationBarrier_Type(W32_SYNCHRONIZATION_BARRIER *lpBarrier);
global W32_DeleteSynchronizationBarrier_Type *w32_DeleteSynchronizationBarrier_func = 0;
typedef BOOL W32_EnterSynchronizationBarrier_Type(W32_SYNCHRONIZATION_BARRIER *lpBarrier, DWORD dwFlags);
global W32_EnterSynchronizationBarrier_Type *w32_EnterSynchronizationBarrier_func = 0;
global RIO_EXTENSION_FUNCTION_TABLE w32_rio_functions = {0};
typedef HRESULT W32_SetThreadDescription_Type(HANDLE hThread, PCWSTR lpThreadDescription);
typedef BOOL W32_InitializeSynchronizationBarrier_Type(W32_SYNCHRONIZATION_BARRIER *lpBarrier, LONG lTotalThreads, LONG lSpinCount);
typedef BOOL W32_DeleteSynchronizationBarrier_Type(W32_SYNCHRONIZATION_BARRIER *lpBarrier);
typedef BOOL W32_EnterSynchronizationBarrier_Type(W32_SYNCHRONIZATION_BARRIER *lpBarrier, DWORD dwFlags);
typedef struct { void *address; SIZE_T size; } W32_PrefetchRange;
typedef BOOL WINAPI W32_PrefetchVirtualMemory_Type(HANDLE, ULONG_PTR, W32_PrefetchRange *, ULONG);
#define W32_KERNEL32_EXT_XLIST \
X(SetThreadDescription) \
X(InitializeSynchronizationBarrier) \
X(DeleteSynchronizationBarrier) \
X(EnterSynchronizationBarrier) \
X(PrefetchVirtualMemory)
typedef PVOID W32_VirtualAlloc2_Type(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER *, ULONG);
typedef PVOID W32_MapViewOfFile3_Type(HANDLE, HANDLE, PVOID, ULONG64, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER *, ULONG);
typedef BOOL W32_UnmapViewOfFile2_Type(HANDLE, PVOID, ULONG);
#define W32_KERNELBASE_EXT_XLIST \
X(VirtualAlloc2) \
X(MapViewOfFile3) \
X(UnmapViewOfFile2)
#define X(n) global W32_##n##_Type *w32_##n##_func;
W32_KERNEL32_EXT_XLIST
W32_KERNELBASE_EXT_XLIST
#undef X
////////////////////////////////
//~ rjf: File Info Conversion Helpers
@@ -304,6 +325,148 @@ release_memory(void *ptr, U64 size)
VirtualFree(ptr, 0, MEM_RELEASE);
}
internal B32
memory_placeholders_supported(void)
{
return w32_VirtualAlloc2_func && w32_MapViewOfFile3_func && w32_UnmapViewOfFile2_func;
}
internal void *
reserve_memory_placeholders(U64 size, U64 block_size)
{
Assert(block_size > 0);
Assert(size > 0);
Assert(size % block_size == 0);
Assert((block_size % get_system_info()->allocation_granularity) == 0);
// reserve requested placeholder block
U8 *base = w32_VirtualAlloc2_func(GetCurrentProcess(), 0, size, MEM_RESERVE|MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, 0, 0);
if (base == 0) { return 0; }
// split placeholder block into specified block sizes
for (U64 offset = 0; offset + block_size < size; offset += block_size)
{
if (VirtualFree(base + offset, block_size, MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER) == 0)
{
if (offset == 0 || VirtualFree(base, size, MEM_RELEASE|MEM_COALESCE_PLACEHOLDERS))
{
VirtualFree(base, 0, MEM_RELEASE);
}
return 0;
}
}
return base;
}
internal void
release_memory_placeholders(void *ptr, U64 size, U64 block_size)
{
Assert(block_size > 0 && size > 0);
if (size > block_size && VirtualFree(ptr, size, MEM_RELEASE|MEM_COALESCE_PLACEHOLDERS) == 0) { return; }
VirtualFree(ptr, 0, MEM_RELEASE);
}
internal void *
shared_memory_view_replace_placeholder(SharedMemory handle, void *ptr, Rng1U64 range, AccessFlags flags)
{
Assert(flags == AccessFlag_Read || flags == (AccessFlag_Read|AccessFlag_Write));
DWORD protection = flags == AccessFlag_Read ? PAGE_READONLY : PAGE_READWRITE;
return w32_MapViewOfFile3_func((HANDLE)handle.u64[0], GetCurrentProcess(), ptr, range.min, dim_1u64(range), MEM_REPLACE_PLACEHOLDER, protection, 0, 0);
}
internal B32
unmap_memory_preserve_placeholder(void *ptr, U64 size)
{
return w32_UnmapViewOfFile2_func(GetCurrentProcess(), ptr, MEM_PRESERVE_PLACEHOLDER);
}
internal B32
split_memory_placeholder(void *ptr, U64 size, U64 first_size)
{
return first_size == size || (first_size && first_size < size && VirtualFree(ptr, first_size, MEM_RELEASE|MEM_PRESERVE_PLACEHOLDER));
}
internal B32
coalesce_memory_placeholders(void *ptr, U64 size)
{
return VirtualFree(ptr, size, MEM_RELEASE|MEM_COALESCE_PLACEHOLDERS) != 0;
}
internal void *
file_map_view_replace_placeholder(FileMap map, void *ptr, Rng1U64 range)
{
return w32_MapViewOfFile3_func((HANDLE)map.u64[0], GetCurrentProcess(), ptr, range.min, dim_1u64(range), MEM_REPLACE_PLACEHOLDER, PAGE_READONLY, 0, 0);
}
internal void
prefetch_memory_ranges(U64 count, Rng1U64 *ranges)
{
W32_PrefetchRange batch[256];
for (U64 first = 0; first < count; first += ArrayCount(batch)) {
U64 n = Min(ArrayCount(batch), count - first);
for EachIndex(i, n) {
batch[i].address = PtrFromInt(ranges[first+i].min);
batch[i].size = dim_1u64(ranges[first+i]);
}
w32_PrefetchVirtualMemory_func(GetCurrentProcess(), n, batch, 0);
}
}
internal LONG CALLBACK
w32_memory_read_fault(EXCEPTION_POINTERS *exception)
{
EXCEPTION_RECORD *record = exception->ExceptionRecord;
if (record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION &&
record->NumberParameters >= 2 &&
record->ExceptionInformation[0] == 0 &&
w32_state.demand_memory.fault((void *)record->ExceptionInformation[1], w32_state.demand_memory.user_data))
{
return EXCEPTION_CONTINUE_EXECUTION;
}
return EXCEPTION_CONTINUE_SEARCH;
}
internal B32
memory_read_fault_handler_set(MemoryReadFaultFunction *func, void *user_data)
{
W32_DemandMemory *memory = &w32_state.demand_memory;
if(func)
{
if (memory->handler == 0)
{
memory->fault = func;
memory->user_data = user_data;
memory->handler = AddVectoredExceptionHandler(1, w32_memory_read_fault);
if(memory->handler == 0)
{
memory->fault = 0;
return 0;
}
}
else { Assert(0 && "handler is already registered"); return 0; }
}
else
{
if (memory->handler)
{
if(RemoveVectoredExceptionHandler(memory->handler) == 0)
{
return 0;
}
memory->handler = 0;
memory->fault = 0;
memory->user_data = 0;
}
else
{ Assert(0 && "handler was not registered"); return 0; }
}
return 1;
}
//- rjf: large pages
internal void *
@@ -333,7 +496,7 @@ shared_memory_alloc(U64 size, String8 name)
PAGE_READWRITE,
(U32)((size & 0xffffffff00000000) >> 32),
(U32)((size & 0x00000000ffffffff)),
(WCHAR *)name16.str);
name.size ? (WCHAR *)name16.str : 0);
SharedMemory result = {(U64)file};
scratch_end(scratch);
return result;
@@ -778,6 +941,7 @@ file_open(AccessFlags flags, String8 path)
if(flags & AccessFlag_ShareWrite) {share_mode |= FILE_SHARE_WRITE|FILE_SHARE_DELETE;}
if(flags & AccessFlag_Write) {creation_disposition = CREATE_ALWAYS;}
if(flags & AccessFlag_Append) {creation_disposition = OPEN_ALWAYS; access_flags |= FILE_APPEND_DATA; }
if(flags & AccessFlag_CreateNew) {creation_disposition = CREATE_NEW;}
if(flags & AccessFlag_Inherited)
{
security_attributes.bInheritHandle = 1;
@@ -983,6 +1147,32 @@ file_reserve_size(File file, U64 size)
return is_reserved;
}
internal B32
file_set_size(File file, U64 size)
{
if (size > max_S64) { return 0; }
FILE_END_OF_FILE_INFO info = {0};
info.EndOfFile.QuadPart = size;
return SetFileInformationByHandle((HANDLE)file.u64[0], FileEndOfFileInfo, &info, sizeof(info));
}
internal B32
file_flush(File file)
{
return FlushFileBuffers((HANDLE)file.u64[0]);
}
internal B32
replace_file_path(String8 dst, String8 src)
{
Temp scratch = scratch_begin(0,0);
String16 dst16 = str16_from_8(scratch.arena, dst);
String16 src16 = str16_from_8(scratch.arena, src);
B32 result = MoveFileExW((WCHAR *)src16.str, (WCHAR *)dst16.str, MOVEFILE_REPLACE_EXISTING|MOVEFILE_WRITE_THROUGH);
scratch_end(scratch);
return result;
}
internal B32
delete_file_at_path(String8 path)
{
@@ -1939,17 +2129,24 @@ w32_entry_point_caller(int argc, WCHAR **wargv)
//- rjf: dynamically load windows functions which are not guaranteed
// in all SDKs
{
HMODULE module = LoadLibraryA("kernel32.dll");
w32_SetThreadDescription_func = (W32_SetThreadDescription_Type *)GetProcAddress(module, "SetThreadDescription");
w32_InitializeSynchronizationBarrier_func = (W32_InitializeSynchronizationBarrier_Type *)GetProcAddress(module, "InitializeSynchronizationBarrier");
w32_DeleteSynchronizationBarrier_func = (W32_DeleteSynchronizationBarrier_Type *)GetProcAddress(module, "DeleteSynchronizationBarrier");
w32_EnterSynchronizationBarrier_func = (W32_EnterSynchronizationBarrier_Type *)GetProcAddress(module, "EnterSynchronizationBarrier");
#define X(n) w32_##n##_func = (void*)GetProcAddress(module, Stringify(n));
HMODULE module;
module = LoadLibraryA("kernel32.dll");
W32_KERNEL32_EXT_XLIST
FreeLibrary(module);
module = LoadLibraryA("kernelbase.dll");
W32_KERNELBASE_EXT_XLIST
FreeLibrary(module);
if(w32_InitializeSynchronizationBarrier_func == 0)
{
w32_DeleteSynchronizationBarrier_func = 0;
w32_EnterSynchronizationBarrier_func = 0;
w32_EnterSynchronizationBarrier_func = 0;
}
FreeLibrary(module);
#undef X
}
//- rjf: try to allow large pages if we can
@@ -2000,11 +2197,16 @@ w32_entry_point_caller(int argc, WCHAR **wargv)
}
}
{
MEMORYSTATUSEX memory = { .dwLength = sizeof(memory) };
GlobalMemoryStatusEx(&memory);
SystemInfo *info = &w32_state.system_info;
info->logical_processor_count = (U64)sysinfo.dwNumberOfProcessors;
info->page_size = sysinfo.dwPageSize;
info->large_page_size = GetLargePageMinimum();
info->allocation_granularity = sysinfo.dwAllocationGranularity;
info->physical_memory_size = memory.ullTotalPhys;
}
{
ProcessInfo *info = &w32_state.process_info;
+12
View File
@@ -88,6 +88,16 @@ struct W32_Entity
};
};
////////////////////////////////
//~ On Demand Memory
typedef struct
{
MemoryReadFaultFunction *fault;
void *user_data;
PVOID handler;
} W32_DemandMemory;
////////////////////////////////
//~ rjf: State
@@ -105,6 +115,8 @@ struct W32_State
CRITICAL_SECTION entity_mutex;
Arena *entity_arena;
W32_Entity *entity_free;
W32_DemandMemory demand_memory;
};
////////////////////////////////
+46
View File
@@ -209,6 +209,52 @@ typedef enum X64_DebugControlFlags
}
X64_DebugControlFlags;
////////////////////////////////
//~ Exception Codes
typedef enum
{
X64_Exception_DE = 0, // Divide error: division by zero or quotient overflow
X64_Exception_DB = 1, // Debug exception (single stepping, hardware breakpoint)
X64_Exception_NMI = 2, // Non-maskable interrupt
X64_Exception_BP = 3, // Breakpoint (int3)
X64_Exception_OF = 4, // Overflow
X64_Exception_BR = 5, // Bound range exceeded
X64_Exception_UD = 6, // Invalid opcode
X64_Exception_NM = 7, // Device not available
X64_Exception_DF = 8, // Double fault
X64_Exception_9 = 9, // Coprocessor segment overrun (obsolete)
X64_Exception_TS = 10, // Invalid task-state segment
X64_Exception_NP = 11, // Segment not present
X64_Exception_SS = 12, // Stack-segment fault
X64_Exception_GP = 13, // General-protection fault
X64_Exception_PF = 14, // Page fault
X64_Exception_15 = 15, // Reserved; Linux names it X86_TRAP_SPURIOUS
X64_Exception_MF = 16, // x87 floating-point exception
X64_Exception_AC = 17, // Alignment check
X64_Exception_MC = 18, // Machine check
X64_Exception_XF = 19, // SIMD floating-point exception
X64_Exception_VE = 20, // Virtualization exception
X64_Exception_CP = 21, // Control-protection exception
X64_Exception_Amd_HV = 28, // Hypervisor injection exception
X64_Exception_Amd_VC = 29, // VMM communication exception
X64_Exception_Amd_SX = 30, // Security exception
} X64_ExceptionCode;
////////////////////////////////
//~ Page Fault Flags
typedef enum
{
X64_PageFaultError_Protection = (1u << 0),
X64_PageFaultError_Write = (1u << 1),
X64_PageFaultError_User = (1u << 2),
X64_PageFaultError_ReservedBit = (1u << 3),
X64_PageFaultError_InstructionFetch = (1u << 4),
X64_PageFaultError_ProtectionKey = (1u << 5),
X64_PageFaultError_ShadowStack = (1u << 6),
} X64_PageFaultError;
////////////////////////////////
//~ rjf: Generated