From 4316325b33248d02b47e354fae8826cd391bc09e Mon Sep 17 00:00:00 2001
From: Ryan Fleury <ryan.j.fleury@gmail.com>
Date: Wed, 19 Mar 2025 16:08:29 -0700
Subject: [PATCH] demon win32: only alloc console when launching target if the
 exe is not a windows gui pe subsystem

---
 src/demon/win32/demon_core_win32.c | 79 +++++++++++++++++++++++++++++-
 src/os/core/os_core.h              |  1 +
 src/raddbg/raddbg_views.c          |  3 +-
 3 files changed, 80 insertions(+), 3 deletions(-)

diff --git a/src/demon/win32/demon_core_win32.c b/src/demon/win32/demon_core_win32.c
index d31d029e..cb0a2515 100644
--- a/src/demon/win32/demon_core_win32.c
+++ b/src/demon/win32/demon_core_win32.c
@@ -1229,6 +1229,75 @@ dmn_ctrl_launch(DMN_CtrlCtx *ctx, OS_ProcessLaunchParams *params)
       cmd = str8_list_join(scratch.arena, &args, &join_params);
     }
     
+    //- rjf: determine if process needs a console
+    B32 needs_console = 1;
+    {
+      String8 exe_path = params->cmd_line.first->string;
+      OS_Handle file = os_file_open(OS_AccessFlag_Read|OS_AccessFlag_ShareRead, exe_path);
+      
+      // rjf: find PE offset
+      U32 pe_offset = 0;
+      {
+        U64 dos_magic_off = 0;
+        U16 dos_magic = 0;
+        os_file_read_struct(file, dos_magic_off, &dos_magic);
+        if(dos_magic == PE_DOS_MAGIC)
+        {
+          U64 pe_offset_off = OffsetOf(PE_DosHeader, coff_file_offset);
+          os_file_read_struct(file, pe_offset_off, &pe_offset);
+        }
+      }
+      
+      // rjf: get COFF header
+      B32 got_coff_header = 0;
+      U64 coff_header_off = 0;
+      COFF_Header coff_header = {0};
+      if(pe_offset > 0)
+      {
+        U64 pe_magic_off = pe_offset;
+        U32 pe_magic = 0;
+        os_file_read_struct(file, pe_magic_off, &pe_magic);
+        if(pe_magic == PE_MAGIC)
+        {
+          coff_header_off = pe_magic_off + sizeof(pe_magic);
+          if(os_file_read_struct(file, coff_header_off, &coff_header))
+          {
+            got_coff_header = 1;
+          }
+        }
+      }
+      
+      // rjf: get subsystem from PE header following COFF header
+      PE_WindowsSubsystem subsystem = 0;
+      {
+        U64 opt_header_off = coff_header_off + sizeof(coff_header);
+        switch(coff_header.machine)
+        {
+          default:{}break;
+          case COFF_MachineType_X64:
+          {
+            PE_OptionalHeader32Plus hdr = {0};
+            os_file_read_struct(file, opt_header_off, &hdr);
+            subsystem = hdr.subsystem;
+          }break;
+          case COFF_MachineType_X86:
+          {
+            PE_OptionalHeader32 hdr = {0};
+            os_file_read_struct(file, opt_header_off, &hdr);
+            subsystem = hdr.subsystem;
+          }break;
+        }
+      }
+      
+      // rjf: determine if we need a console depending on the subsystem
+      if(subsystem == PE_WindowsSubsystem_WINDOWS_GUI)
+      {
+        needs_console = 0;
+      }
+      
+      os_file_close(file);
+    }
+    
     //- rjf: produce environment strings
     String8 env = {0};
     {
@@ -1291,7 +1360,10 @@ dmn_ctrl_launch(DMN_CtrlCtx *ctx, OS_ProcessLaunchParams *params)
       inherit_handles = 1;
     }
     PROCESS_INFORMATION process_info = {0};
-    AllocConsole();
+    if(needs_console)
+    {
+      AllocConsole();
+    }
     if(CreateProcessW(0, (WCHAR*)cmd16.str, 0, 0, 1, creation_flags, (WCHAR*)env16.str, (WCHAR*)dir16.str, &startup_info, &process_info))
     {
       // check if we are 32-bit app, and just close it immediately
@@ -1315,7 +1387,10 @@ dmn_ctrl_launch(DMN_CtrlCtx *ctx, OS_ProcessLaunchParams *params)
     {
       MessageBox(0, "Error starting process.", "Process error", MB_OK|MB_ICONSTOP);
     }
-    FreeConsole();
+    if(needs_console)
+    {
+      FreeConsole();
+    }
     
     //- rjf: eliminate all handles which have stuck around from the AllocConsole
     {
diff --git a/src/os/core/os_core.h b/src/os/core/os_core.h
index 8edba0ea..cf0e3000 100644
--- a/src/os/core/os_core.h
+++ b/src/os/core/os_core.h
@@ -202,6 +202,7 @@ internal void os_abort(S32 exit_code);
 internal OS_Handle      os_file_open(OS_AccessFlags flags, String8 path);
 internal void           os_file_close(OS_Handle file);
 internal U64            os_file_read(OS_Handle file, Rng1U64 rng, void *out_data);
+#define os_file_read_struct(f, off, ptr) os_file_read((f), r1u64((off), (off)+sizeof(*(ptr))), (ptr))
 internal U64            os_file_write(OS_Handle file, Rng1U64 rng, void *data);
 internal B32            os_file_set_times(OS_Handle file, DateTime time);
 internal FileProperties os_properties_from_file(OS_Handle file);
diff --git a/src/raddbg/raddbg_views.c b/src/raddbg/raddbg_views.c
index a54b9342..08ead8b1 100644
--- a/src/raddbg/raddbg_views.c
+++ b/src/raddbg/raddbg_views.c
@@ -1731,7 +1731,8 @@ RD_VIEW_UI_FUNCTION_DEF(text)
       Temp scratch = scratch_begin(0, 0);
       UI_PrefWidth(ui_children_sum(1)) UI_PrefHeight(ui_em(3, 1))
         UI_Row UI_Padding(ui_pct(1, 0))
-        UI_PrefWidth(ui_text_dim(10, 1))
+        UI_PrefWidth(ui_text_dim(1, 1))
+        UI_TagF("weak")
       {
         RD_Font(RD_FontSlot_Icons) ui_label(rd_icon_kind_text_table[RD_IconKind_WarningBig]);
         ui_labelf("Could not find \"%S\".", rd_regs()->file_path);