fix: work around VRCF's reflective hacks to ensure correct processing…

… order (#126)

This works around some of VRCFury's horrible hacks to improve compatibility between
NDMF and VRCF. In particular, we detect when VRCFury is invoking us reflectively, and
either skip optimizations (if running in play mode), or ignore VRCFury's invocation and
allow VRChat's build hooks to run us (in a build).

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 - Adjusted hook processing order to improve compatibility with VRCFury (#122)
+- Worked around a hack in VRCFury that broke optimization plugins (#126)
 
 ### Removed
 

Editor/ApplyOnPlay.cs

@@ -62,7 +62,9 @@ private static void MaybeProcessAvatar(ApplyOnPlayGlobalActivator.OnDemandSource
             {
                 var avatar = RuntimeUtil.FindAvatarInParents(component.transform);
                 if (avatar == null) return;
-                AvatarProcessor.ProcessAvatar(avatar.gameObject);
+
+                // Skip optimizing the avatar as we might have VRCFury or similar running after us.
+                AvatarProcessor.ProcessAvatar(avatar.gameObject, BuildPhase.Transforming);
             }
         }
 

Editor/AvatarProcessor.cs

@@ -2,6 +2,7 @@
 
 using System;
 using System.Diagnostics;
+using System.Linq;
 using nadena.dev.ndmf.runtime;
 using UnityEditor;
 using UnityEngine;
@@ -98,23 +99,81 @@ public static GameObject ProcessAvatarUI(GameObject obj)
             }
         }
 
+        #if NDMF_VRCSDK3_AVATARS
+        private static bool IsVRCFuryHack(System.Diagnostics.StackTrace trace)
+        {
+            foreach (var frame in trace.GetFrames())
+            {
+                Debug.Log("Frame: " + frame.GetMethod().DeclaringType.FullName + " " + frame.GetMethod());
+            }
+
+            return trace.GetFrames().Any(frame =>
+                frame.GetMethod().DeclaringType.FullName == "VF.Menu.NdmfFirstMenuItem"
+            );
+        }
+
+        private static bool InHookExecution(System.Diagnostics.StackTrace trace)
+        {
+            return trace.GetFrames().Any(frame =>
+                typeof(VRC.SDKBase.Editor.BuildPipeline.IVRCSDKPreprocessAvatarCallback)
+                    .IsAssignableFrom(frame.GetMethod().DeclaringType));
+        }
+        #else
+        private static bool IsVRCFuryHack(System.Diagnostics.StackTrace trace)
+        {
+            return false;
+        }
+
+        private static bool InHookExecution(System.Diagnostics.StackTrace trace)
+        {
+            return false;
+        }
+        #endif
+
         /// <summary>
         /// Processes an avatar as part of an automated process. The resulting assets will be saved in a temporary
         /// location.
         /// </summary>
         /// <param name="root"></param>
         public static void ProcessAvatar(GameObject root)
         {
-            if (root.GetComponent<AlreadyProcessedTag>()) return;
-
+            ProcessAvatar(root, BuildPhase.Optimizing);
+        }
+
+        internal static void ProcessAvatar(GameObject root, BuildPhase lastPhase) {
+            if (root.GetComponent<AlreadyProcessedTag>()?.processingCompleted == true) return;
+
+            // HACK: VRCFury tries to invoke ProcessAvatar during its own processing, but this risks having Optimization
+            // phase passes run too early (before VRCF runs). Detect when we're being invoked like this and skip
+            // optimization.
+            System.Diagnostics.StackTrace stackTrace = new System.Diagnostics.StackTrace();
+            if (IsVRCFuryHack(stackTrace))
+            {
+                if (InHookExecution(stackTrace))
+                {
+                    Debug.Log("NDMF: Detected VRCFury hack from within VRChat build hooks - " +
+                              "ignoring VRCFury invocation");
+                    // We're running from within VRChat build hooks, so just ignore VRCFury's request;
+                    // we'll be run in the correct order anyway.
+                    return;
+                }
+                else
+                {
+                    Debug.Log("NDMF: Detected VRCFury hack from play mode - skipping optimization");
+                    // Skip optimizations, because they might break VRCFury processing.
+                    lastPhase = BuildPhase.Transforming;
+                }
+            }
+
             var buildContext = new BuildContext(root, TemporaryAssetRoot);
 
-            ProcessAvatar(buildContext, BuildPhase.Resolving, BuildPhase.Optimizing);
+            ProcessAvatar(buildContext, BuildPhase.Resolving, lastPhase);
             buildContext.Finish();
 
             if (RuntimeUtil.IsPlaying)
             {
-                root.AddComponent<AlreadyProcessedTag>();
+                var tag = root.GetComponent<AlreadyProcessedTag>() ?? root.AddComponent<AlreadyProcessedTag>();
+                tag.processingCompleted = true;
             }
         }
 

Editor/VRChat/BuildFrameworkPreprocessHook.cs

@@ -27,7 +27,7 @@ internal class BuildFrameworkPreprocessHook : IVRCSDKPreprocessAvatarCallback
 
         public bool OnPreprocessAvatar(GameObject avatarGameObject)
         {
-            if (avatarGameObject.GetComponent<AlreadyProcessedTag>()) return true;
+            if (avatarGameObject.GetComponent<AlreadyProcessedTag>()?.processingCompleted == true) return true;
 
             try
             {
@@ -52,7 +52,7 @@ internal class BuildFrameworkOptimizeHook : IVRCSDKPreprocessAvatarCallback
 
         public bool OnPreprocessAvatar(GameObject avatarGameObject)
         {
-            if (avatarGameObject.GetComponent<AlreadyProcessedTag>()) return true;
+            if (avatarGameObject.GetComponent<AlreadyProcessedTag>()?.processingCompleted == true) return true;
 
             var holder = avatarGameObject.GetComponent<ContextHolder>();
             if (holder == null) return true;

Runtime/AlreadyProcessedTag.cs

@@ -6,6 +6,10 @@ namespace nadena.dev.ndmf.runtime
     [AddComponentMenu("")]
     internal class AlreadyProcessedTag : MonoBehaviour
     {
+        // VRCF creates this tag via reflection, but we're not actually done processing yet.
+        // We add this boolean so we can ignore any tags created surrepitiously by VRCF...
+        internal bool processingCompleted;
+
         private void OnValidate()
         {
             hideFlags = HideFlags.HideAndDontSave;