ruby: Use cleaner OpenGL render pattern, misc. CGL fixes (#1543)

### OpenGL rendering (all platforms)

- librashader specifies that the output viewport should be the same size
as the output framebuffer texture. This was not the case for the OpenGL
render code.
* Previously, librashader under OpenGL would render onto an intermediate
framebuffer sized to the "output" size (the entire window view size),
rather than the "target" size (the final composited size of the game
area within the output area). The `libra_viewport_t` would be sized to
the target size, but the underlying buffer would be larger.
* Now, we size the intermediate framebuffer to the "target" size, let
librashader render onto it with a `libra_viewport_t` that matches that
size. In the final pass we sample this buffer within an area of the
"output"-sized buffer as appropriate.

This prior behavior would lead to scaling issues with shaders in the
Metal backend. The same issues did not seem to be obviously present in
OpenGL in my testing, but we should nevertheless probably fix this in
case it is causing any of the subtle issues with shaders that have been
reported, and also in case something breaks in the future as a result of
not following this recommendation.

(The above is also unrelated to the scaling issues addressed by
#1508)

### CGL fix-ups (macOS)

- Backports the native fullscreen and monitor selection options to
OpenGL, and removes unnecessary custom window code from the OpenGL
driver.
- Removes a seemingly unnecessary output call from `reshape` that would
cause flickering during resizes.

This has been tested on macOS but should probably be tested on other
platforms as well to make sure nothing breaks.

Co-authored-by: jcm <[email protected]>

ruby/video/cgl.cpp

@@ -12,15 +12,6 @@ struct VideoCGL;
 -(BOOL) acceptsFirstResponder;
 @end
 
-@interface RubyWindowCGL : NSWindow <NSWindowDelegate> {
-@public
-  VideoCGL* video;
-}
--(id) initWith:(VideoCGL*)video;
--(BOOL) canBecomeKeyWindow;
--(BOOL) canBecomeMainWindow;
-@end
-
 struct VideoCGL : VideoDriver, OpenGL {
   VideoCGL& self = *this;
   VideoCGL(Video& super) : VideoDriver(super) {}
@@ -34,14 +25,52 @@ struct VideoCGL : VideoDriver, OpenGL {
   auto ready() -> bool override { return _ready; }
 
   auto hasFullScreen() -> bool override { return true; }
+  auto hasNativeFullScreen() -> bool override { return true; }
+  auto hasMonitor() -> bool override { return !_nativeFullScreen; }
   auto hasContext() -> bool override { return true; }
   auto hasBlocking() -> bool override { return true; }
   auto hasForceSRGB() -> bool override { return false; }
   auto hasFlush() -> bool override { return true; }
   auto hasShader() -> bool override { return true; }
 
   auto setFullScreen(bool fullScreen) -> bool override {
-    return initialize();
+    // todo: fix/make consistent mouse cursor hide behavior
+
+    if (_nativeFullScreen) {
+      [view.window toggleFullScreen:nil];
+    } else {
+      /// This option implements non-idiomatic macOS fullscreen behavior that sets the window frame equal to the selected display's
+      /// frame size and hides the cursor. This version of fullscreen is desirable because it allows us to render around the camera
+      /// housing on newer Macs (important for bezel-style shaders), has snappier entrance/exit and tabbing behavior, and functions
+      /// better with recording and capture software such as OBS.
+      if (fullScreen) {
+        auto monitor = Video::monitor(self.monitor);
+        NSScreen *handle = (__bridge NSScreen *)(void *)monitor.nativeHandle; //eew
+        frameBeforeFullScreen = view.window.frame;
+        [NSApp setPresentationOptions:(NSApplicationPresentationAutoHideDock | NSApplicationPresentationAutoHideMenuBar)];
+        [view.window setStyleMask:NSWindowStyleMaskBorderless];
+        [view.window setFrame:handle.frame display:YES];
+        [NSCursor setHiddenUntilMouseMoves:YES];
+      } else {
+        [NSApp setPresentationOptions:NSApplicationPresentationDefault];
+        [view.window setStyleMask:(NSWindowStyleMaskTitled | NSWindowStyleMaskMiniaturizable | NSWindowStyleMaskResizable | NSWindowStyleMaskClosable)];
+        [view.window setFrame:frameBeforeFullScreen display:YES];
+      }
+      [view.window makeFirstResponder:view];
+    }
+    return true;
+  }
+
+  auto setNativeFullScreen(bool nativeFullScreen) -> bool override {
+    _nativeFullScreen = nativeFullScreen;
+    if (nativeFullScreen) {
+      //maximize goes fullscreen
+      [view.window setCollectionBehavior: NSWindowCollectionBehaviorFullScreenPrimary];
+    } else {
+      //maximize does not go fullscreen
+      [view.window setCollectionBehavior: NSWindowCollectionBehaviorFullScreenAuxiliary];
+    }
+    return true;
   }
 
   auto setContext(uintptr context) -> bool override {
@@ -135,13 +164,6 @@ struct VideoCGL : VideoDriver, OpenGL {
     terminate();
     if(!self.fullScreen && !self.context) return false;
 
-    if(self.fullScreen) {
-      window = [[RubyWindowCGL alloc] initWith:this];
-      [window setCollectionBehavior:NSWindowCollectionBehaviorFullScreenPrimary];
-      [window toggleFullScreen:nil];
-    //[NSApp setPresentationOptions:NSApplicationPresentationFullScreen];
-    }
-
     NSOpenGLPixelFormatAttribute attributeList[] = {
       NSOpenGLPFAOpenGLProfile, NSOpenGLProfileVersion3_2Core,
       NSOpenGLPFAColorSize, 24,
@@ -150,7 +172,7 @@ struct VideoCGL : VideoDriver, OpenGL {
       0
     };
 
-    auto context = self.fullScreen ? [window contentView] : (__bridge NSView*)(void *)self.context;
+    auto context = (__bridge NSView*)(void *)self.context;
     auto size = [context frame].size;
     auto format = [[NSOpenGLPixelFormat alloc] initWithAttributes:attributeList];
     auto openGLContext = [[NSOpenGLContext alloc] initWithFormat:format shareContext:nil];
@@ -176,6 +198,7 @@ struct VideoCGL : VideoDriver, OpenGL {
 
     releaseContext();
     clear();
+    setNativeFullScreen(_nativeFullScreen);
     return _ready = true;
   }
 
@@ -188,19 +211,12 @@ struct VideoCGL : VideoDriver, OpenGL {
       [view removeFromSuperview];
       view = nil;
     }
-
-    if(window) {
-    //[NSApp setPresentationOptions:NSApplicationPresentationDefault];
-      [window toggleFullScreen:nil];
-      [window setCollectionBehavior:NSWindowCollectionBehaviorDefault];
-      [window close];
-      window = nil;
-    }
   }
 
   RubyVideoCGL* view = nullptr;
-  RubyWindowCGL* window = nullptr;
 
+  bool _nativeFullScreen = false;
+  NSRect frameBeforeFullScreen = NSMakeRect(0,0,0,0);
   bool _ready = false;
   std::recursive_mutex mutex;
 };
@@ -216,7 +232,6 @@ struct VideoCGL : VideoDriver, OpenGL {
 
 -(void) reshape {
   [super reshape];
-  video->output(0, 0);
 }
 
 -(BOOL) acceptsFirstResponder {
@@ -230,28 +245,3 @@ struct VideoCGL : VideoDriver, OpenGL {
 }
 
 @end
-
-@implementation RubyWindowCGL : NSWindow
-
--(id) initWith:(VideoCGL*)videoPointer {
-  auto primaryRect = [[[NSScreen screens] objectAtIndex:0] frame];
-  if(self = [super initWithContentRect:primaryRect styleMask:0 backing:NSBackingStoreBuffered defer:YES]) {
-    video = videoPointer;
-    [self setDelegate:self];
-    [self setReleasedWhenClosed:NO];
-    [self setAcceptsMouseMovedEvents:YES];
-    [self setTitle:@""];
-    [self makeKeyAndOrderFront:nil];
-  }
-  return self;
-}
-
--(BOOL) canBecomeKeyWindow {
-  return YES;
-}
-
--(BOOL) canBecomeMainWindow {
-  return YES;
-}
-
-@end

ruby/video/metal/metal.cpp

@@ -586,6 +586,7 @@ struct VideoMetal : VideoDriver, Metal {
     }
 
     initialized = true;
+    setNativeFullScreen(self.nativeFullScreen);
     return _ready = true;
   }
 

ruby/video/opengl/main.hpp

@@ -67,8 +67,8 @@ auto OpenGL::output() -> void {
   u32 y = (outputHeight - targetHeight) / 2;
 
   if(_chain != NULL) {
-    // Shader path: our intermediate framebuffer matches the output size
-    if(!framebuffer || framebufferWidth != outputWidth || framebufferHeight != outputHeight) {
+    // Shader path: our intermediate framebuffer matches the target size (final composited game area size)
+    if(!framebuffer || framebufferWidth != targetWidth || framebufferHeight != targetHeight) {
       if(framebuffer) {
         glDeleteFramebuffers(1, &framebuffer);
         framebuffer = 0;
@@ -78,7 +78,7 @@ auto OpenGL::output() -> void {
         framebufferTexture = 0;
       }
 
-      framebufferWidth = outputWidth, framebufferHeight = outputHeight;
+      framebufferWidth = targetWidth, framebufferHeight = targetHeight;
       glGenFramebuffers(1, &framebuffer);
       glBindFramebuffer(GL_FRAMEBUFFER, framebuffer);
       glGenTextures(1, &framebufferTexture);

ruby/video/opengl/surface.hpp

@@ -24,7 +24,7 @@ auto OpenGLSurface::render(u32 sourceWidth, u32 sourceHeight, u32 targetX, u32 t
 
   if(_chain != NULL) {
     libra_source_image_gl_t input = {texture, format, sourceWidth, sourceHeight};
-    libra_viewport_t viewport{(float)targetX, (float)targetY, targetWidth, targetHeight};
+    libra_viewport_t viewport{0, 0, targetWidth, targetHeight};
     libra_output_framebuffer_gl_t output = {framebuffer, framebufferTexture, framebufferFormat};
 
     if (auto error = _libra.gl_filter_chain_frame(&_chain, frameCount++, input, viewport, output, NULL, NULL)) {
@@ -33,7 +33,7 @@ auto OpenGLSurface::render(u32 sourceWidth, u32 sourceHeight, u32 targetX, u32 t
 
     glBindFramebuffer(GL_READ_FRAMEBUFFER, framebuffer);
     glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
-    glBlitFramebuffer(0, framebufferHeight, framebufferWidth, 0, 0, 0, framebufferWidth, framebufferHeight, GL_COLOR_BUFFER_BIT, filter);
+    glBlitFramebuffer(0, framebufferHeight, framebufferWidth, 0, targetX, targetY, framebufferWidth + targetX, framebufferHeight + targetY, GL_COLOR_BUFFER_BIT, filter);
     glBindFramebuffer(GL_READ_FRAMEBUFFER, 0);
   } else {
     glBindFramebuffer(GL_READ_FRAMEBUFFER, framebuffer);