Improvements to render tests (#2160)

- Display image statistics when image differencing is enabled in render tests. - Add supersampling to hardware shading languages when reference quality is enabled. - Improve camera alignment across shading languages. - Remove legacy render test options for clarity.
AcademySoftwareFoundation · Dec 29, 2024 · 32ff290 · 32ff290
1 parent 8ba7530
commit 32ff290
Show file tree

Hide file tree

Showing 9 changed files with 99 additions and 196 deletions.
diff --git a/python/MaterialXTest/tests_to_html.bat b/python/MaterialXTest/tests_to_html.bat
@@ -1,2 +1,2 @@
 @echo off
-python tests_to_html.py -i1 ../../build %*
+python tests_to_html.py -i1 ../../build %* -d
diff --git a/python/MaterialXTest/tests_to_html.py b/python/MaterialXTest/tests_to_html.py
@@ -6,13 +6,13 @@
 import argparse
 
 try:
-    # Use pip to install Pillow and Image to enable image diffs
-    from PIL import Image, ImageChops
+    # Install pillow via pip to enable image differencing and statistics.
+    from PIL import Image, ImageChops, ImageStat
     DIFF_ENABLED = True
 except Exception:
     DIFF_ENABLED = False
 
-def createDiff(image1Path, image2Path, imageDiffPath):
+def computeDiff(image1Path, image2Path, imageDiffPath):
     try:
         if os.path.exists(imageDiffPath):
             os.remove(imageDiffPath)
@@ -29,6 +29,8 @@ def createDiff(image1Path, image2Path, imageDiffPath):
         image2 = Image.open(image2Path).convert('RGB')
         diff = ImageChops.difference(image1, image2)
         diff.save(imageDiffPath)
+        diffStat = ImageStat.Stat(diff)
+        return sum(diffStat.rms) / (3.0 * 255.0)
     except Exception:
         if os.path.exists(imageDiffPath):
             os.remove(imageDiffPath)
@@ -144,9 +146,8 @@ def main(args=None):
             fullPath1 = os.path.join(path1, file1) if file1 else None
             fullPath2 = os.path.join(path2, file2) if file2 else None
             fullPath3 = os.path.join(path3, file3) if file3 else None
-            diffPath1 = None
-            diffPath2 = None
-            diffPath3 = None
+            diffPath1 = diffPath2 = diffPath3 = None
+            diffRms1 = diffRms2 = diffRms3 = None
 
             if curPath != path1:
                 if curPath != "":
@@ -157,13 +158,13 @@ def main(args=None):
 
             if file1 and file2 and DIFF_ENABLED and args.CREATE_DIFF:
                 diffPath1 = fullPath1[0:-8] + "_" + args.lang1 + "-1_vs_" + args.lang2 + "-2_diff.png"
-                createDiff(fullPath1, fullPath2, diffPath1)
+                diffRms1 = computeDiff(fullPath1, fullPath2, diffPath1)
 
             if useThirdLang and file1 and file3 and DIFF_ENABLED and args.CREATE_DIFF:
                 diffPath2 = fullPath1[0:-8] + "_" + args.lang1 + "-1_vs_" + args.lang3 + "-3_diff.png"
-                createDiff(fullPath1, fullPath3, diffPath2)
+                diffRms2 = computeDiff(fullPath1, fullPath3, diffPath2)
                 diffPath3 = fullPath1[0:-8] + "_" + args.lang2 + "-2_vs_" + args.lang3 + "-3_diff.png"
-                createDiff(fullPath2, fullPath3, diffPath3)
+                diffRms3 = computeDiff(fullPath2, fullPath3, diffPath3)
 
             def prependFileUri(filepath: str) -> str:
                 if os.path.isabs(filepath):
@@ -203,11 +204,11 @@ def prependFileUri(filepath: str) -> str:
                     fh.write("<br>(" + str(datetime.datetime.fromtimestamp(os.path.getmtime(fullPath3))) + ")")
                 fh.write("</td>\n")
             if diffPath1:
-                fh.write("<td align='center'>Difference " + args.lang1 + " vs. " + args.lang2 + " </td>\n")
+                fh.write("<td align='center'>" + args.lang1.upper() + " vs. " + args.lang2.upper() + " (RMS " + "%.5f" % diffRms1 + ") </td>\n")
             if diffPath2:
-                fh.write("<td align='center'>Difference " + args.lang1 + " vs. " + args.lang3 + " </td>\n")
+                fh.write("<td align='center'>" + args.lang1.upper() + " vs. " + args.lang3.upper() + " (RMS " + "%.5f" % diffRms2 + ") </td>\n")
             if diffPath3:
-                fh.write("<td align='center'>Difference " + args.lang2 + " vs. " + args.lang3 + " </td>\n")
+                fh.write("<td align='center'>" + args.lang2.upper() + " vs. " + args.lang3.upper() + " (RMS " + "%.5f" % diffRms3 + ") </td>\n")
             fh.write("</tr>\n")
 
     fh.write("</table>\n")

diff --git a/resources/Materials/TestSuite/_options.mtlx b/resources/Materials/TestSuite/_options.mtlx
@@ -33,21 +33,9 @@
     -->
     <input name="shaderInterfaces" type="integer" value="2" />
 
-    <!-- Validate element before attempting to generate code. Default is false  -->
-    <input name="validateElementToRender" type="boolean" value="false" />
-
-    <!-- Perform source code compilation validation test -->
-    <input name="compileCode" type="boolean" value="true" />
-
-    <!-- Perform rendering validation test -->
-    <input name="renderImages" type="boolean" value="true" />
-
     <!-- Rendered image size if render validation test enabled -->
     <input name="renderSize" type="vector2" value="512, 512" />
 
-    <!-- Perform saving of image. Can only be disabled for GLSL tests -->
-    <input name="saveImages" type="boolean" value="true" />
-
     <!-- Set this to be true if it is desired to dump out uniform and attribut information to the logging file -->
     <input name="dumpUniformsAndAttributes" type="boolean" value="true" />
 
@@ -76,7 +64,11 @@
     <!-- List of document paths for render tests -->
     <input name="renderTestPaths" type="string" value="resources/Materials/Examples/StandardSurface,resources/Materials/TestSuite/stdlib/color_management,resources/Materials/TestSuite/stdlib/convolution,resources/Materials/TestSuite/stdlib/geometric,resources/Materials/TestSuite/stdlib/procedural,resources/Materials/TestSuite/pbrlib,resources/Materials/TestSuite/nprlib" />
 
-    <!-- Enable reference quality rendering. Default is false. -->
+    <!-- Enable reference quality rendering.
+         This option enables higher sample counts and supersampling in render tests,
+         allowing for visual comparisons and differencing across shading languages,
+         but requiring a more powerful GPU and longer CPU render times.
+    -->
     <input name="enableReferenceQuality" type="boolean" value="false" />
   </nodedef>
 </materialx>
diff --git a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp
@@ -921,11 +921,7 @@ void TestSuiteOptions::print(std::ostream& output) const
     output << "\tCheck Implementation Usage Count: " << checkImplCount << std::endl;
     output << "\tDump Generated Code: " << dumpGeneratedCode << std::endl;
     output << "\tShader Interfaces: " << shaderInterfaces << std::endl;
-    output << "\tValidate Element To Render: " << validateElementToRender << std::endl;
-    output << "\tCompile code: " << compileCode << std::endl;
-    output << "\tRender Images: " << renderImages << std::endl;
     output << "\tRender Size: " << renderSize[0] << "," << renderSize[1] << std::endl;
-    output << "\tSave Images: " << saveImages << std::endl;
     output << "\tDump uniforms and Attributes  " << dumpUniformsAndAttributes << std::endl;
     output << "\tRender Geometry: " << renderGeometry.asString() << std::endl;
     output << "\tEnable Direct Lighting: " << enableDirectLighting << std::endl;
@@ -947,11 +943,7 @@ bool TestSuiteOptions::readOptions(const std::string& optionFile)
     const std::string TARGETS_STRING("targets");
     const std::string LIGHT_FILES_STRING("lightFiles");
     const std::string SHADER_INTERFACES_STRING("shaderInterfaces");
-    const std::string VALIDATE_ELEMENT_TO_RENDER_STRING("validateElementToRender");
-    const std::string COMPILE_CODE_STRING("compileCode");
-    const std::string RENDER_IMAGES_STRING("renderImages");
     const std::string RENDER_SIZE_STRING("renderSize");
-    const std::string SAVE_IMAGES_STRING("saveImages");
     const std::string DUMP_UNIFORMS_AND_ATTRIBUTES_STRING("dumpUniformsAndAttributes");
     const std::string CHECK_IMPL_COUNT_STRING("checkImplCount");
     const std::string DUMP_GENERATED_CODE_STRING("dumpGeneratedCode");
@@ -998,26 +990,10 @@ bool TestSuiteOptions::readOptions(const std::string& optionFile)
                     {
                         shaderInterfaces = val->asA<int>();
                     }
-                    else if (name == VALIDATE_ELEMENT_TO_RENDER_STRING)
-                    {
-                        validateElementToRender = val->asA<bool>();
-                    }
-                    else if (name == COMPILE_CODE_STRING)
-                    {
-                        compileCode = val->asA<bool>();
-                    }
-                    else if (name == RENDER_IMAGES_STRING)
-                    {
-                        renderImages = val->asA<bool>();
-                    }
                     else if (name == RENDER_SIZE_STRING)
                     {
                         renderSize = val->asA<mx::Vector2>();
                     }
-                    else if (name == SAVE_IMAGES_STRING)
-                    {
-                        saveImages = val->asA<bool>();
-                    }
                     else if (name == DUMP_UNIFORMS_AND_ATTRIBUTES_STRING)
                     {
                         dumpUniformsAndAttributes = val->asA<bool>();
@@ -1082,23 +1058,11 @@ bool TestSuiteOptions::readOptions(const std::string& optionFile)
             }
         }
 
-        // Disable render and save of images if not compiled code will be generated
-        if (!compileCode)
-        {
-            renderImages = false;
-            saveImages = false;
-        }
-        // Disable saving images, if no images are to be produced
-        if (!renderImages)
-        {
-            saveImages = false;
-        }
-        // Disable direct lighting
+        // Handle direct and indirect lighting toggles.
         if (!enableDirectLighting)
         {
             lightFiles.clear();
         }
-        // Disable indirect lighting
         if (!enableIndirectLighting)
         {
             radianceIBLPath.assign(mx::EMPTY_STRING);

diff --git a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.h b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.h
@@ -88,21 +88,9 @@ class TestSuiteOptions
     // - 1 = run reduced only.
     int shaderInterfaces = 2;
 
-    // Validate element before attempting to generate code. Default is false.
-    bool validateElementToRender = false;
-
-    // Perform source code compilation validation test
-    bool compileCode = true;
-
-    // Perform rendering validation test
-    bool renderImages = true;
-
     // Render size
     mx::Vector2 renderSize = { 512, 512 };
 
-    // Perform saving of image.
-    bool saveImages = true;
-
     // Set this to be true if it is desired to dump out uniform and attribut information to the logging file.
     bool dumpUniformsAndAttributes = true;
 

diff --git a/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp b/source/MaterialXTest/MaterialXRenderGlsl/RenderGlsl.cpp
@@ -168,17 +168,6 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName,
     const mx::ShaderGenerator& shadergen = context.getShaderGenerator();
     mx::FileSearchPath searchPath = mx::getDefaultDataSearchPath();
 
-    // Perform validation if requested
-    if (testOptions.validateElementToRender)
-    {
-        std::string message;
-        if (!element->validate(&message))
-        {
-            log << "Element is invalid: " << message << std::endl;
-            return false;
-        }
-    }
-
     std::vector<mx::GenOptions> optionsList;
     getGenerationOptions(testOptions, context.getOptions(), optionsList);
 
@@ -247,11 +236,6 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName,
                 file.close();
             }
 
-            if (!testOptions.compileCode)
-            {
-                return false;
-            }
-
             // Validate
             MaterialX::GlslProgramPtr program = _renderer->getProgram();
             bool validated = false;
@@ -356,27 +340,31 @@ bool GlslShaderRenderTester::runRenderer(const std::string& shaderName,
                     }
                 }
 
-                if (testOptions.renderImages)
+                int supersampleFactor = testOptions.enableReferenceQuality ? 8 : 1;
+
                 {
-                    {
-                        mx::ScopedTimer renderTimer(&profileTimes.languageTimes.renderTime);
-                        _renderer->getImageHandler()->setSearchPath(imageSearchPath);
-                        _renderer->setSize(static_cast<unsigned int>(testOptions.renderSize[0]), static_cast<unsigned int>(testOptions.renderSize[1]));
-                        _renderer->render();
-                    }
+                    mx::ScopedTimer renderTimer(&profileTimes.languageTimes.renderTime);
+                    _renderer->getImageHandler()->setSearchPath(imageSearchPath);
+                    unsigned int width = (unsigned int) testOptions.renderSize[0] * supersampleFactor;
+                    unsigned int height = (unsigned int) testOptions.renderSize[1] * supersampleFactor;
+                    _renderer->setSize(width, height);
+                    _renderer->render();
+                }
 
-                    if (testOptions.saveImages)
+                {
+                    mx::ScopedTimer ioTimer(&profileTimes.languageTimes.imageSaveTime);
+                    std::string fileName = shaderPath + "_glsl.png";
+                    mx::ImagePtr image = _renderer->captureImage();
+                    if (image)
                     {
-                        mx::ScopedTimer ioTimer(&profileTimes.languageTimes.imageSaveTime);
-                        std::string fileName = shaderPath + "_glsl.png";
-                        mx::ImagePtr image = _renderer->captureImage();
-                        if (image)
+                        if (supersampleFactor > 1)
                         {
-                            _renderer->getImageHandler()->saveImage(fileName, image, true);
-                            if (imageVec)
-                            {
-                                imageVec->push_back(image);
-                            }
+                            image = image->applyBoxDownsample(supersampleFactor);
+                        }
+                        _renderer->getImageHandler()->saveImage(fileName, image, true);
+                        if (imageVec)
+                        {
+                            imageVec->push_back(image);
                         }
                     }
                 }

diff --git a/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm b/source/MaterialXTest/MaterialXRenderMsl/RenderMsl.mm
@@ -104,6 +104,7 @@ bool runRenderer(const std::string& shaderName,
     _lightHandler->setEnvRadianceMap(envRadiance);
     _lightHandler->setEnvIrradianceMap(envIrradiance);
     _lightHandler->setEnvSampleCount(options.enableReferenceQuality ? 4096 : 1024);
+    _lightHandler->setRefractionTwoSided(true);
 }
 
 //
@@ -175,17 +176,6 @@ bool runRenderer(const std::string& shaderName,
 
     const mx::ShaderGenerator& shadergen = context.getShaderGenerator();
 
-    // Perform validation if requested
-    if (testOptions.validateElementToRender)
-    {
-        std::string message;
-        if (!element->validate(&message))
-        {
-            log << "Element is invalid: " << message << std::endl;
-            return false;
-        }
-    }
-
     std::vector<mx::GenOptions> optionsList;
     getGenerationOptions(testOptions, context.getOptions(), optionsList);
 
@@ -254,11 +244,6 @@ bool runRenderer(const std::string& shaderName,
                 file.close();
             }
 
-            if (!testOptions.compileCode)
-            {
-                return false;
-            }
-
             // Validate
             bool validated = false;
             try
@@ -362,27 +347,31 @@ bool runRenderer(const std::string& shaderName,
                     }
                 }
 
-                if (testOptions.renderImages)
+                int supersampleFactor = testOptions.enableReferenceQuality ? 8 : 1;
+
                 {
-                    {
-                        mx::ScopedTimer renderTimer(&profileTimes.languageTimes.renderTime);
-                        _renderer->getImageHandler()->setSearchPath(mx::getDefaultDataSearchPath());
-                        _renderer->setSize(static_cast<unsigned int>(testOptions.renderSize[0]), static_cast<unsigned int>(testOptions.renderSize[1]));
-                        _renderer->render();
-                    }
+                    mx::ScopedTimer renderTimer(&profileTimes.languageTimes.renderTime);
+                    _renderer->getImageHandler()->setSearchPath(mx::getDefaultDataSearchPath());
+                    unsigned int width = (unsigned int) testOptions.renderSize[0] * supersampleFactor;
+                    unsigned int height = (unsigned int) testOptions.renderSize[1] * supersampleFactor;
+                    _renderer->setSize(width, height);
+                    _renderer->render();
+                }
 
-                    if (testOptions.saveImages)
+                {
+                    mx::ScopedTimer ioTimer(&profileTimes.languageTimes.imageSaveTime);
+                    std::string fileName = shaderPath + "_msl.png";
+                    mx::ImagePtr image = _renderer->captureImage();
+                    if (image)
                     {
-                        mx::ScopedTimer ioTimer(&profileTimes.languageTimes.imageSaveTime);
-                        std::string fileName = shaderPath + "_msl.png";
-                        mx::ImagePtr image = _renderer->captureImage();
-                        if (image)
+                        if (supersampleFactor > 1)
                         {
-                            _renderer->getImageHandler()->saveImage(fileName, image, true);
-                            if (imageVec)
-                            {
-                                imageVec->push_back(image);
-                            }
+                            image = image->applyBoxDownsample(supersampleFactor);
+                        }
+                        _renderer->getImageHandler()->saveImage(fileName, image, true);
+                        if (imageVec)
+                        {
+                            imageVec->push_back(image);
                         }
                     }
                 }