diff --git a/GPU/Common/FramebufferCommon.cpp b/GPU/Common/FramebufferCommon.cpp
index 8cba489400b5..447c10a26199 100644
--- a/GPU/Common/FramebufferCommon.cpp
+++ b/GPU/Common/FramebufferCommon.cpp
@@ -544,6 +544,20 @@ void FramebufferManagerCommon::UpdateFromMemory(u32 addr, int size, bool safe) {
 	}
 }
 
+void FramebufferManagerCommon::DownloadFramebufferOnSwitch(VirtualFramebuffer *vfb) {
+	if (vfb && vfb->safeWidth > 0 && vfb->safeHeight > 0 && !vfb->firstFrameSaved) {
+		// Some games will draw to some memory once, and use it as a render-to-texture later.
+		// To support this, we save the first frame to memory when we have a save w/h.
+		// Saving each frame would be slow.
+		if (!g_Config.bDisableSlowFramebufEffects) {
+			ReadFramebufferToMemory(vfb, true, 0, 0, vfb->safeWidth, vfb->safeHeight);
+			vfb->firstFrameSaved = true;
+			vfb->safeWidth = 0;
+			vfb->safeHeight = 0;
+		}
+	}
+}
+
 bool FramebufferManagerCommon::NotifyFramebufferCopy(u32 src, u32 dst, int size, bool isMemset, u32 skipDrawReason) {
 	if (updateVRAM_ || size == 0) {
 		return false;
diff --git a/GPU/Common/FramebufferCommon.h b/GPU/Common/FramebufferCommon.h
index 9fe53073e282..8a13b4400257 100644
--- a/GPU/Common/FramebufferCommon.h
+++ b/GPU/Common/FramebufferCommon.h
@@ -62,6 +62,7 @@ struct VirtualFramebuffer {
 	u32 clutUpdatedBytes;
 	bool memoryUpdated;
 	bool depthUpdated;
+	bool firstFrameSaved;
 
 	u32 fb_address;
 	u32 z_address;
@@ -252,6 +253,7 @@ class FramebufferManagerCommon {
 	void ShowScreenResolution();
 
 	bool ShouldDownloadFramebuffer(const VirtualFramebuffer *vfb) const;
+	void DownloadFramebufferOnSwitch(VirtualFramebuffer *vfb);
 	void FindTransferFramebuffers(VirtualFramebuffer *&dstBuffer, VirtualFramebuffer *&srcBuffer, u32 dstBasePtr, int dstStride, int &dstX, int &dstY, u32 srcBasePtr, int srcStride, int &srcX, int &srcY, int &srcWidth, int &srcHeight, int &dstWidth, int &dstHeight, int bpp) const;
 	VirtualFramebuffer *FindDownloadTempBuffer(VirtualFramebuffer *vfb);
 	virtual bool CreateDownloadTempBuffer(VirtualFramebuffer *nvfb) = 0;
diff --git a/GPU/Directx9/FramebufferDX9.cpp b/GPU/Directx9/FramebufferDX9.cpp
index 66af073c10bb..50ce4dc18353 100644
--- a/GPU/Directx9/FramebufferDX9.cpp
+++ b/GPU/Directx9/FramebufferDX9.cpp
@@ -401,6 +401,8 @@ namespace DX9 {
 	void FramebufferManagerDX9::NotifyRenderFramebufferSwitched(VirtualFramebuffer *prevVfb, VirtualFramebuffer *vfb, bool isClearingDepth) {
 		if (ShouldDownloadFramebuffer(vfb) && !vfb->memoryUpdated) {
 			ReadFramebufferToMemory(vfb, true, 0, 0, vfb->width, vfb->height);
+		} else {
+			DownloadFramebufferOnSwitch(prevVfb);
 		}
 		textureCache_->ForgetLastTexture();
 
@@ -704,6 +706,8 @@ namespace DX9 {
 	}
 
 	void FramebufferManagerDX9::CopyDisplayToOutput() {
+		DownloadFramebufferOnSwitch(currentRenderVfb_);
+
 		fbo_unbind();
 		currentRenderVfb_ = 0;
 
@@ -1218,9 +1222,6 @@ namespace DX9 {
 			if (vfb != displayFramebuf_ && vfb != prevDisplayFramebuf_ && vfb != prevPrevDisplayFramebuf_) {
 				if (age > FBO_OLD_AGE) {
 					INFO_LOG(SCEGE, "Decimating FBO for %08x (%i x %i x %i), age %i", vfb->fb_address, vfb->width, vfb->height, vfb->format, age);
-					if (!g_Config.bDisableSlowFramebufEffects && vfb->safeWidth > 0 && vfb->safeHeight > 0) {
-						ReadFramebufferToMemory(vfb, true, 0, 0, vfb->safeWidth, vfb->safeHeight);
-					}
 					DestroyFramebuf(vfb);
 					vfbs_.erase(vfbs_.begin() + i--);
 				}
@@ -1269,12 +1270,6 @@ namespace DX9 {
 		for (size_t i = 0; i < vfbs_.size(); ++i) {
 			VirtualFramebuffer *vfb = vfbs_[i];
 			INFO_LOG(SCEGE, "Destroying FBO for %08x : %i x %i x %i", vfb->fb_address, vfb->width, vfb->height, vfb->format);
-			if (!forceDelete && !g_Config.bDisableSlowFramebufEffects && vfb->safeWidth > 0 && vfb->safeHeight > 0) {
-				// But also let's check if Memory is shut down already.
-				if (Memory::IsActive()) {
-					ReadFramebufferToMemory(vfb, true, 0, 0, vfb->safeWidth, vfb->safeHeight);
-				}
-			}
 			DestroyFramebuf(vfb);
 		}
 		vfbs_.clear();
diff --git a/GPU/GLES/Framebuffer.cpp b/GPU/GLES/Framebuffer.cpp
index 4cf69c259923..0db902128386 100644
--- a/GPU/GLES/Framebuffer.cpp
+++ b/GPU/GLES/Framebuffer.cpp
@@ -654,6 +654,8 @@ void FramebufferManager::NotifyRenderFramebufferCreated(VirtualFramebuffer *vfb)
 void FramebufferManager::NotifyRenderFramebufferSwitched(VirtualFramebuffer *prevVfb, VirtualFramebuffer *vfb, bool isClearingDepth) {
 	if (ShouldDownloadFramebuffer(vfb) && !vfb->memoryUpdated) {
 		ReadFramebufferToMemory(vfb, true, 0, 0, vfb->width, vfb->height);
+	} else {
+		DownloadFramebufferOnSwitch(prevVfb);
 	}
 	textureCache_->ForgetLastTexture();
 
@@ -911,9 +913,10 @@ struct CardboardSettings * FramebufferManager::GetCardboardSettings(struct Cardb
 }
 
 void FramebufferManager::CopyDisplayToOutput() {
-	fbo_unbind();
-	glstate.viewport.set(0, 0, pixelWidth_, pixelHeight_);
+	DownloadFramebufferOnSwitch(currentRenderVfb_);
 
+	glstate.viewport.set(0, 0, pixelWidth_, pixelHeight_);
+	fbo_unbind();
 	currentRenderVfb_ = 0;
 
 	if (displayFramebufPtr_ == 0) {
@@ -1895,9 +1898,6 @@ void FramebufferManager::DecimateFBOs() {
 		if (vfb != displayFramebuf_ && vfb != prevDisplayFramebuf_ && vfb != prevPrevDisplayFramebuf_) {
 			if (age > FBO_OLD_AGE) {
 				INFO_LOG(SCEGE, "Decimating FBO for %08x (%i x %i x %i), age %i", vfb->fb_address, vfb->width, vfb->height, vfb->format, age);
-				if (!g_Config.bDisableSlowFramebufEffects && vfb->safeWidth > 0 && vfb->safeHeight > 0) {
-					ReadFramebufferToMemory(vfb, true, 0, 0, vfb->safeWidth, vfb->safeHeight);
-				}
 				DestroyFramebuf(vfb);
 				vfbs_.erase(vfbs_.begin() + i--);
 			}
@@ -1936,12 +1936,6 @@ void FramebufferManager::DestroyAllFBOs(bool forceDelete) {
 	for (size_t i = 0; i < vfbs_.size(); ++i) {
 		VirtualFramebuffer *vfb = vfbs_[i];
 		INFO_LOG(SCEGE, "Destroying FBO for %08x : %i x %i x %i", vfb->fb_address, vfb->width, vfb->height, vfb->format);
-		if (!forceDelete && !g_Config.bDisableSlowFramebufEffects && vfb->safeWidth > 0 && vfb->safeHeight > 0) {
-			// But also let's check if Memory is shut down already.
-			if (Memory::IsActive()) {
-				ReadFramebufferToMemory(vfb, true, 0, 0, vfb->safeWidth, vfb->safeHeight);
-			}
-		}
 		DestroyFramebuf(vfb);
 	}
 	vfbs_.clear();
diff --git a/GPU/Vulkan/FramebufferVulkan.cpp b/GPU/Vulkan/FramebufferVulkan.cpp
index 4aba64abd1ae..9cda132f25d6 100644
--- a/GPU/Vulkan/FramebufferVulkan.cpp
+++ b/GPU/Vulkan/FramebufferVulkan.cpp
@@ -628,6 +628,8 @@ void FramebufferManagerVulkan::NotifyRenderFramebufferCreated(VirtualFramebuffer
 void FramebufferManagerVulkan::NotifyRenderFramebufferSwitched(VirtualFramebuffer *prevVfb, VirtualFramebuffer *vfb, bool isClearingDepth) {
 	if (ShouldDownloadFramebuffer(vfb) && !vfb->memoryUpdated) {
 		ReadFramebufferToMemory(vfb, true, 0, 0, vfb->width, vfb->height);
+	} else {
+		DownloadFramebufferOnSwitch(prevVfb);
 	}
 	textureCache_->ForgetLastTexture();
 
@@ -819,8 +821,10 @@ void FramebufferManagerVulkan::CopyDisplayToOutput() {
 	// then in theory, we can even avoid starting up a render pass at all for the backbuffer (!). not sure if that
 	// is worth the needed refactoring trouble though.
 
-	// fbo_unbind();
 
+	DownloadFramebufferOnSwitch(currentRenderVfb_);
+
+	// fbo_unbind();
 	currentRenderVfb_ = 0;
 
 	if (useBufferedRendering_) {
@@ -1551,9 +1555,6 @@ void FramebufferManagerVulkan::DecimateFBOs() {
 		if (vfb != displayFramebuf_ && vfb != prevDisplayFramebuf_ && vfb != prevPrevDisplayFramebuf_) {
 			if (age > FBO_OLD_AGE) {
 				INFO_LOG(SCEGE, "Decimating FBO for %08x (%i x %i x %i), age %i", vfb->fb_address, vfb->width, vfb->height, vfb->format, age);
-				if (!g_Config.bDisableSlowFramebufEffects && vfb->safeWidth > 0 && vfb->safeHeight > 0) {
-					ReadFramebufferToMemory(vfb, true, 0, 0, vfb->safeWidth, vfb->safeHeight);
-				}
 				DestroyFramebuf(vfb);
 				vfbs_.erase(vfbs_.begin() + i--);
 			}
@@ -1570,12 +1571,6 @@ void FramebufferManagerVulkan::DestroyAllFBOs(bool forceDelete) {
 	for (size_t i = 0; i < vfbs_.size(); ++i) {
 		VirtualFramebuffer *vfb = vfbs_[i];
 		INFO_LOG(SCEGE, "Destroying FBO for %08x : %i x %i x %i", vfb->fb_address, vfb->width, vfb->height, vfb->format);
-		if (!forceDelete && !g_Config.bDisableSlowFramebufEffects && vfb->safeWidth > 0 && vfb->safeHeight > 0) {
-			// But also let's check if Memory is shut down already.
-			if (Memory::IsActive()) {
-				ReadFramebufferToMemory(vfb, true, 0, 0, vfb->safeWidth, vfb->safeHeight);
-			}
-		}
 		DestroyFramebuf(vfb);
 	}
 	vfbs_.clear();