Do not over-commit fiber stacks on Windows

src/crystal/system/fiber.cr

@@ -1,12 +1,12 @@
 module Crystal::System::Fiber
   # Allocates memory for a stack.
-  # def self.allocate_stack(stack_size : Int) : Void*
+  # def self.allocate_stack(stack_size : Int, protect : Bool) : Void*
+
+  # Prepares an existing, unused stack for use again.
+  # def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil
 
   # Frees memory of a stack.
   # def self.free_stack(stack : Void*, stack_size : Int) : Nil
-
-  # Determines location of the top of the main process fiber's stack.
-  # def self.main_fiber_stack(stack_bottom : Void*) : Void*
 end
 
 {% if flag?(:wasi) %}

src/crystal/system/unix/fiber.cr

@@ -21,6 +21,9 @@ module Crystal::System::Fiber
     pointer
   end
 
+  def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil
+  end
+
   def self.free_stack(stack : Void*, stack_size) : Nil
     LibC.munmap(stack, stack_size)
   end

src/crystal/system/wasi/fiber.cr

@@ -3,6 +3,9 @@ module Crystal::System::Fiber
     LibC.malloc(stack_size)
   end
 
+  def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil
+  end
+
   def self.free_stack(stack : Void*, stack_size) : Nil
     LibC.free(stack)
   end

src/crystal/system/win32/fiber.cr

@@ -7,28 +7,61 @@ module Crystal::System::Fiber
   # overflow
   RESERVED_STACK_SIZE = LibC::DWORD.new(0x10000)
 
-  # the reserved stack size, plus the size of a single page
-  @@total_reserved_size : LibC::DWORD = begin
-    LibC.GetNativeSystemInfo(out system_info)
-    system_info.dwPageSize + RESERVED_STACK_SIZE
-  end
-
   def self.allocate_stack(stack_size, protect) : Void*
-    unless memory_pointer = LibC.VirtualAlloc(nil, stack_size, LibC::MEM_COMMIT | LibC::MEM_RESERVE, LibC::PAGE_READWRITE)
-      raise RuntimeError.from_winerror("VirtualAlloc")
+    if stack_top = LibC.VirtualAlloc(nil, stack_size, LibC::MEM_RESERVE, LibC::PAGE_READWRITE)
+      if protect
+        if commit_and_guard(stack_top, stack_size)
+          return stack_top
+        end
+      else
+        # for the interpreter, the stack is just ordinary memory so the entire
+        # range is committed
+        if LibC.VirtualAlloc(stack_top, stack_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE)
+          return stack_top
+        end
+      end
+
+      # failure
+      LibC.VirtualFree(stack_top, 0, LibC::MEM_RELEASE)
     end
 
-    # Detects stack overflows by guarding the top of the stack, similar to
-    # `LibC.mprotect`. Windows will fail to allocate a new guard page for these
-    # fiber stacks and trigger a stack overflow exception
+    raise RuntimeError.from_winerror("VirtualAlloc")
+  end
+
+  def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil
     if protect
-      if LibC.VirtualProtect(memory_pointer, @@total_reserved_size, LibC::PAGE_READWRITE | LibC::PAGE_GUARD, out _) == 0
-        LibC.VirtualFree(memory_pointer, 0, LibC::MEM_RELEASE)
-        raise RuntimeError.from_winerror("VirtualProtect")
+      if LibC.VirtualFree(stack, 0, LibC::MEM_DECOMMIT) == 0
+        raise RuntimeError.from_winerror("VirtualFree")
+      end
+      unless commit_and_guard(stack, stack_size)
+        raise RuntimeError.from_winerror("VirtualAlloc")
       end
     end
+  end
 
-    memory_pointer
+  # Commits the bottommost page and sets up the guard pages above it, in the
+  # same manner as each thread's main stack. When the stack hits a guard page
+  # for the first time, a page fault is generated, the page's guard status is
+  # reset, and Windows checks if a reserved page is available above. On success,
+  # a new guard page is committed, and on failure, a stack overflow exception is
+  # triggered after the `RESERVED_STACK_SIZE` portion is made available.
+  private def self.commit_and_guard(stack_top, stack_size)
+    stack_bottom = stack_top + stack_size
+
+    LibC.GetNativeSystemInfo(out system_info)
+    stack_commit_size = system_info.dwPageSize
+    stack_commit_top = stack_bottom - stack_commit_size
+    if LibC.VirtualAlloc(stack_commit_top, stack_commit_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE) == 0
+      return false
+    end
+
+    # the reserved stack size, plus a final guard page for when the stack
+    # overflow handler itself overflows the stack
+    stack_guard_size = system_info.dwPageSize + RESERVED_STACK_SIZE
+    stack_guard_top = stack_commit_top - stack_guard_size
+    if LibC.VirtualAlloc(stack_guard_top, stack_guard_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE | LibC::PAGE_GUARD) == 0
+      return false
+    end
   end
 
   def self.free_stack(stack : Void*, stack_size) : Nil

src/fiber/stack_pool.cr

@@ -42,7 +42,11 @@ class Fiber
 
     # Removes a stack from the bottom of the pool, or allocates a new one.
     def checkout : {Void*, Void*}
-      stack = @deque.pop? || Crystal::System::Fiber.allocate_stack(STACK_SIZE, @protect)
+      if stack = @deque.pop?
+        Crystal::System::Fiber.reset_stack(stack, STACK_SIZE, @protect)
+      else
+        stack = Crystal::System::Fiber.allocate_stack(STACK_SIZE, @protect)
+      end
       {stack, stack + STACK_SIZE}
     end