From 94b43c6dee0f9bc9e828a6db88a245f22d32140c Mon Sep 17 00:00:00 2001
From: "Unknown W. Brackets" <checkins@unknownbrackets.org>
Date: Mon, 5 Dec 2022 23:14:41 -0800
Subject: [PATCH] softgpu: Avoid tri combine to rect if clipping.

---
 GPU/Software/RasterizerRectangle.cpp | 29 ++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/GPU/Software/RasterizerRectangle.cpp b/GPU/Software/RasterizerRectangle.cpp
index 2ce43cba8901..9241c4bf6032 100644
--- a/GPU/Software/RasterizerRectangle.cpp
+++ b/GPU/Software/RasterizerRectangle.cpp
@@ -392,6 +392,19 @@ bool RectangleFastPath(const VertexData &v0, const VertexData &v1, BinManager &b
 	return false;
 }
 
+static bool IsCoordRectangleCompatible(const RasterizerState &state, const ClipVertexData &data) {
+	if (!state.throughMode) {
+		// See AreCoordsRectangleCompatible() for most of these, this just checks the main vert.
+		if (data.OutsideRange())
+			return false;
+		if (data.clippos.w < 0.0f)
+			return false;
+		if (data.clippos.z < -data.clippos.w)
+			return false;
+	}
+	return true;
+}
+
 static bool AreCoordsRectangleCompatible(const RasterizerState &state, const ClipVertexData &data0, const ClipVertexData &data1) {
 	if (data1.v.color0 != data0.v.color0)
 		return false;
@@ -404,7 +417,7 @@ static bool AreCoordsRectangleCompatible(const RasterizerState &state, const Cli
 		if (data1.v.color1 != data0.v.color1)
 			return false;
 		// This means it should be culled, outside range.
-		if (data1.OutsideRange() || data0.OutsideRange())
+		if (data1.OutsideRange())
 			return false;
 		// Do we have to think about perspective correction or slope mip level?
 		if (state.enableTextures && data1.clippos.w != data0.clippos.w) {
@@ -414,7 +427,10 @@ static bool AreCoordsRectangleCompatible(const RasterizerState &state, const Cli
 				return false;
 		}
 		// We might need to cull this if all verts have negative w, which doesn't seem to happen for rectangles.
-		if (data0.clippos.w < 0.0f && data1.clippos.w < 0.0f)
+		if (data1.clippos.w < 0.0f)
+			return false;
+		// And we also may need to clip, even if flat.
+		if (data1.clippos.z < -data1.clippos.w)
 			return false;
 		// If we're projecting textures, only allow an exact match for simplicity.
 		if (state.enableTextures && data1.v.texturecoords.q() != data0.v.texturecoords.q())
@@ -430,6 +446,9 @@ static bool AreCoordsRectangleCompatible(const RasterizerState &state, const Cli
 }
 
 bool DetectRectangleFromStrip(const RasterizerState &state, const ClipVertexData data[4], int *tlIndex, int *brIndex) {
+	if (!IsCoordRectangleCompatible(state, data[0]))
+		return false;
+
 	// Color and Z must be flat.  Also find the TL and BR meanwhile.
 	int tl = 0, br = 0;
 	for (int i = 1; i < 4; ++i) {
@@ -486,6 +505,9 @@ bool DetectRectangleFromStrip(const RasterizerState &state, const ClipVertexData
 }
 
 bool DetectRectangleFromFan(const RasterizerState &state, const ClipVertexData *data, int *tlIndex, int *brIndex) {
+	if (!IsCoordRectangleCompatible(state, data[0]))
+		return false;
+
 	// Color and Z must be flat.
 	int tl = 0, br = 0;
 	for (int i = 1; i < 4; ++i) {
@@ -537,6 +559,9 @@ bool DetectRectangleFromFan(const RasterizerState &state, const ClipVertexData *
 }
 
 bool DetectRectangleFromPair(const RasterizerState &state, const ClipVertexData data[6], int *tlIndex, int *brIndex) {
+	if (!IsCoordRectangleCompatible(state, data[0]))
+		return false;
+
 	// Color and Z must be flat.  Also find the TL and BR meanwhile.
 	int tl = 0, br = 0;
 	for (int i = 1; i < 6; ++i) {