Do not overcommit fiber stacks on Windows

src/crystal/system/win32/fiber.cr

@@ -7,28 +7,42 @@ module Crystal::System::Fiber
   # overflow
   RESERVED_STACK_SIZE = LibC::DWORD.new(0x10000)
 
-  # the reserved stack size, plus the size of a single page
-  @@total_reserved_size : LibC::DWORD = begin
-    LibC.GetNativeSystemInfo(out system_info)
-    system_info.dwPageSize + RESERVED_STACK_SIZE
-  end
-
   def self.allocate_stack(stack_size, protect) : Void*
-    unless memory_pointer = LibC.VirtualAlloc(nil, stack_size, LibC::MEM_COMMIT | LibC::MEM_RESERVE, LibC::PAGE_READWRITE)
-      raise RuntimeError.from_winerror("VirtualAlloc")
-    end
+    if stack_top = LibC.VirtualAlloc(nil, stack_size, LibC::MEM_RESERVE, LibC::PAGE_READWRITE)
+      if protect
+        # Commits the bottommost page and sets up the guard pages above it, in
+        # the same manner as each thread's main stack. When the stack hits an
+        # uncommitted guard page, Windows checks if a reserved page is available
+        # above, then commits and guards one page each on success, or triggers a
+        # stack overflow exception on failure. The guard pages serve a similar
+        # purpose as `LibC.mprotect`
+        stack_bottom = stack_top + stack_size
+
+        LibC.GetNativeSystemInfo(out system_info)
+        stack_commit_size = system_info.dwPageSize
+        stack_commit_top = stack_bottom - stack_commit_size
+
+        # the reserved stack size, plus a final guard page for when the stack
+        # overflow handler itself overflows the stack
+        stack_guard_size = system_info.dwPageSize + RESERVED_STACK_SIZE
+        stack_guard_top = stack_commit_top - stack_guard_size
 
-    # Detects stack overflows by guarding the top of the stack, similar to
-    # `LibC.mprotect`. Windows will fail to allocate a new guard page for these
-    # fiber stacks and trigger a stack overflow exception
-    if protect
-      if LibC.VirtualProtect(memory_pointer, @@total_reserved_size, LibC::PAGE_READWRITE | LibC::PAGE_GUARD, out _) == 0
-        LibC.VirtualFree(memory_pointer, 0, LibC::MEM_RELEASE)
-        raise RuntimeError.from_winerror("VirtualProtect")
+        if LibC.VirtualAlloc(stack_commit_top, stack_commit_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE)
+          if LibC.VirtualAlloc(stack_guard_top, stack_guard_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE | LibC::PAGE_GUARD)
+            return stack_top
+          end
+        end
+      else
+        # for the interpreter, the stack is just ordinary memory so the entire
+        # range is committed
+        if LibC.VirtualAlloc(stack_top, stack_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE)
+          return stack_top
+        end
       end
     end
 
-    memory_pointer
+    LibC.VirtualFree(stack_top, 0, LibC::MEM_RELEASE) if stack_top
+    raise RuntimeError.from_winerror("VirtualAlloc")
   end
 
   def self.free_stack(stack : Void*, stack_size) : Nil