From 13cdcc5ce0b7ca0b9fa592e77707234d96adf0ca Mon Sep 17 00:00:00 2001
From: Barnaby <22575741+barnabwhy@users.noreply.github.com>
Date: Wed, 14 Feb 2024 00:49:57 +0000
Subject: [PATCH] Fix pre-analysis setting for AMF (#1985)

* actually obey pre-analysis setting

* give warnings about latency when the encoder enables pre-analysis

* nvm actually enable pre-analysis with the setting lol

* remove typo
---
 .../cpp/platform/win32/VideoEncoderAMF.cpp    | 43 +++++++++++++------
 alvr/server/src/connection.rs                 |  2 +-
 2 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/alvr/server/cpp/platform/win32/VideoEncoderAMF.cpp b/alvr/server/cpp/platform/win32/VideoEncoderAMF.cpp
index a0aab57c88..7dc5524fc6 100644
--- a/alvr/server/cpp/platform/win32/VideoEncoderAMF.cpp
+++ b/alvr/server/cpp/platform/win32/VideoEncoderAMF.cpp
@@ -210,10 +210,16 @@ amf::AMFComponentPtr VideoEncoderAMF::MakeEncoder(
 			caps->GetProperty(AMF_VIDEO_ENCODER_CAP_PRE_ANALYSIS, &m_hasPreAnalysis);
 			caps->GetProperty(AMF_VIDEO_ENCODER_CAPS_QUERY_TIMEOUT_SUPPORT, &m_hasQueryTimeout);
 		}
-		if (m_hasPreAnalysis) {
-			amfEncoder->SetProperty(AMF_VIDEO_ENCODER_PRE_ANALYSIS_ENABLE, Settings::Instance().m_enablePreAnalysis);
-		} else {
-			Warn("Pre-analysis could not be enabled because your GPU does not support it for h264 encoding.");
+
+		if (Settings::Instance().m_enablePreAnalysis) {
+			if (!Settings::Instance().m_usePreproc || Settings::Instance().m_use10bitEncoder) {
+				Warn("Pre-analysis could not be enabled because \"Use preproc\" is not enabled or \"Reduce color banding\" is enabled.");
+			} else if (m_hasPreAnalysis) {
+				Warn("Enabling h264 pre-analysis. You may experience higher latency when this is enabled.");
+				amfEncoder->SetProperty(AMF_VIDEO_ENCODER_PRE_ANALYSIS_ENABLE, Settings::Instance().m_enablePreAnalysis);
+			} else {
+				Warn("Pre-analysis could not be enabled because your GPU does not support it for h264 encoding.");
+			}
 		}
 
 		//No noticable performance difference and should improve subjective quality by allocating more bits to smooth areas
@@ -281,10 +287,16 @@ amf::AMFComponentPtr VideoEncoderAMF::MakeEncoder(
 			caps->GetProperty(AMF_VIDEO_ENCODER_HEVC_CAP_PRE_ANALYSIS, &m_hasPreAnalysis);
 			caps->GetProperty(AMF_VIDEO_ENCODER_CAPS_HEVC_QUERY_TIMEOUT_SUPPORT, &m_hasQueryTimeout);
 		}
-		if (m_hasPreAnalysis) {
-			amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_PRE_ANALYSIS_ENABLE, Settings::Instance().m_enablePreAnalysis);
-		} else {
-			Warn("Pre-analysis could not be enabled because your GPU does not support it for HEVC encoding.");
+
+		if (Settings::Instance().m_enablePreAnalysis) {
+			if (!Settings::Instance().m_usePreproc || Settings::Instance().m_use10bitEncoder) {
+				Warn("Pre-analysis could not be enabled because \"Use preproc\" is not enabled or \"Reduce color banding\" is enabled.");
+			} else if (m_hasPreAnalysis) {
+				Warn("Enabling HEVC pre-analysis. You may experience higher latency when this is enabled.");
+				amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_PRE_ANALYSIS_ENABLE, Settings::Instance().m_enablePreAnalysis);
+			} else {
+				Warn("Pre-analysis could not be enabled because your GPU does not support it for HEVC encoding.");
+			}
 		}
 
 		//No noticable performance difference and should improve subjective quality by allocating more bits to smooth areas
@@ -361,11 +373,16 @@ amf::AMFComponentPtr VideoEncoderAMF::MakeEncoder(
 		if (amfEncoder->GetCaps(&caps) == AMF_OK) {
 			caps->GetProperty(AMF_VIDEO_ENCODER_AV1_CAP_PRE_ANALYSIS, &m_hasPreAnalysis);
 		}
-		if (m_hasPreAnalysis) {
-			Warn("Enabling AV1 pre-analysis.");
-			amfEncoder->SetProperty(AMF_VIDEO_ENCODER_AV1_PRE_ANALYSIS_ENABLE, Settings::Instance().m_enablePreAnalysis);
-		} else {
-			Warn("Pre-analysis could not be enabled because your GPU does not support it for AV1 encoding.");
+		
+		if (Settings::Instance().m_enablePreAnalysis) {
+			if (!Settings::Instance().m_usePreproc || Settings::Instance().m_use10bitEncoder) {
+				Warn("Pre-analysis could not be enabled because \"Use preproc\" is not enabled or \"Reduce color banding\" is enabled.");
+			} else if (m_hasPreAnalysis) {
+				Warn("Enabling AV1 pre-analysis. You may experience higher latency when this is enabled.");
+				amfEncoder->SetProperty(AMF_VIDEO_ENCODER_AV1_PRE_ANALYSIS_ENABLE, Settings::Instance().m_enablePreAnalysis);
+			} else {
+				Warn("Pre-analysis could not be enabled because your GPU does not support it for AV1 encoding.");
+			}
 		}
 
 		// May impact performance but improves quality in high-motion areas
diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs
index 3167d52ce2..f1e9d870fc 100644
--- a/alvr/server/src/connection.rs
+++ b/alvr/server/src/connection.rs
@@ -170,7 +170,7 @@ pub fn contruct_openvr_config(session: &SessionConfig) -> OpenvrConfig {
         filler_data: settings.video.encoder_config.filler_data,
         entropy_coding: settings.video.encoder_config.entropy_coding as u32,
         use_10bit_encoder: settings.video.encoder_config.use_10bit,
-        // enable_pre_analysis: amf_controls.enable_pre_analysis,
+        enable_pre_analysis: amf_controls.enable_pre_analysis,
         enable_vbaq: amf_controls.enable_vbaq,
         enable_hmqb: amf_controls.enable_hmqb,
         use_preproc: amf_controls.use_preproc,