diff --git a/Source/Scene/ImageBasedLighting.js b/Source/Scene/ImageBasedLighting.js
index 52255febb1ee..4e3d27de51a6 100644
--- a/Source/Scene/ImageBasedLighting.js
+++ b/Source/Scene/ImageBasedLighting.js
@@ -1,16 +1,11 @@
 import Cartesian2 from "../Core/Cartesian2.js";
 import Check from "../Core/Check.js";
-import Matrix3 from "../Core/Matrix3.js";
-import Matrix4 from "../Core/Matrix4.js";
 import defined from "../Core/defined.js";
 import defaultValue from "../Core/defaultValue.js";
 import destroyObject from "../Core/destroyObject.js";
 import DeveloperError from "../Core/DeveloperError.js";
 import OctahedralProjectedCubeMap from "./OctahedralProjectedCubeMap.js";
 
-const scratchIBLReferenceFrameMatrix4 = new Matrix4();
-const scratchIBLReferenceFrameMatrix3 = new Matrix3();
-
 /**
  * Properties for managing image-based lighting on tilesets and models.
  * Also manages the necessary resources and textures.
@@ -98,11 +93,6 @@ export default function ImageBasedLighting(options) {
   this._useDefaultSphericalHarmonics = false;
   this._shouldRegenerateShaders = false;
 
-  // The model/tileset will set this with a setter after construction.
-  this._referenceMatrix = Matrix3.clone(Matrix3.IDENTITY);
-  // Derived from the reference matrix and the current view matrix
-  this._iblReferenceFrameMatrix = Matrix3.clone(Matrix3.IDENTITY);
-
   // Store the previous frame number to prevent redundant update calls
   this._previousFrameNumber = undefined;
 
@@ -244,39 +234,6 @@ Object.defineProperties(ImageBasedLighting.prototype, {
     },
   },
 
-  /**
-   * The reference frame to use for transforming the image-based lighting. This
-   * is either the model's modelMatrix, or another reference matrix (e.g. the
-   * tileset modelMatrix).
-   *
-   * @memberof ImageBasedLighting.prototype
-   * @type {Matrix4}
-   *
-   * @private
-   */
-  referenceMatrix: {
-    get: function () {
-      return this._referenceMatrix;
-    },
-    set: function (value) {
-      this._referenceMatrix = value;
-    },
-  },
-
-  /**
-   * The reference frame for image-based lighting, derived from the reference matrix and the current view matrix.
-   *
-   * @memberof ImageBasedLighting.prototype
-   * @type {Matrix3}
-   *
-   * @private
-   */
-  iblReferenceFrameMatrix: {
-    get: function () {
-      return this._iblReferenceFrameMatrix;
-    },
-  },
-
   /**
    * Whether or not image-based lighting is enabled.
    *
@@ -510,32 +467,6 @@ ImageBasedLighting.prototype.update = function (frameState) {
   this._useDefaultSphericalHarmonics =
     !defined(this._sphericalHarmonicCoefficients) &&
     defined(frameState.sphericalHarmonicCoefficients);
-
-  const useSHC = this.useSphericalHarmonicCoefficients;
-  const useSEM = this.useSpecularEnvironmentMaps;
-
-  if (useSHC || useSEM) {
-    let iblReferenceFrameMatrix3 = scratchIBLReferenceFrameMatrix3;
-    let iblReferenceFrameMatrix4 = scratchIBLReferenceFrameMatrix4;
-
-    iblReferenceFrameMatrix4 = Matrix4.multiply(
-      context.uniformState.view3D,
-      this._referenceMatrix,
-      iblReferenceFrameMatrix4
-    );
-    iblReferenceFrameMatrix3 = Matrix4.getMatrix3(
-      iblReferenceFrameMatrix4,
-      iblReferenceFrameMatrix3
-    );
-    iblReferenceFrameMatrix3 = Matrix3.getRotation(
-      iblReferenceFrameMatrix3,
-      iblReferenceFrameMatrix3
-    );
-    this._iblReferenceFrameMatrix = Matrix3.transpose(
-      iblReferenceFrameMatrix3,
-      this._iblReferenceFrameMatrix
-    );
-  }
 };
 
 /**
diff --git a/Source/Scene/Model.js b/Source/Scene/Model.js
index 528576dcb8da..5634ebc7a5a7 100644
--- a/Source/Scene/Model.js
+++ b/Source/Scene/Model.js
@@ -613,6 +613,7 @@ function Model(options) {
 
   this._computedModelMatrix = new Matrix4(); // Derived from modelMatrix and scale
   this._clippingPlanesMatrix = Matrix4.clone(Matrix4.IDENTITY); // Derived from reference matrix and the current view matrix
+  this._iblReferenceFrameMatrix = Matrix3.clone(Matrix3.IDENTITY); // Derived from reference matrix and the current view matrix
   this._initialRadius = undefined; // Radius without model's scale property, model-matrix scale, animations, or skins
   this._boundingSphere = undefined;
   this._scaledBoundingSphere = new BoundingSphere();
@@ -3768,7 +3769,7 @@ function createClippingPlanesMatrixFunction(model) {
 
 function createIBLReferenceFrameMatrixFunction(model) {
   return function () {
-    return model._imageBasedLighting.iblReferenceFrameMatrix;
+    return model._iblReferenceFrameMatrix;
   };
 }
 
@@ -5219,6 +5220,8 @@ function distanceDisplayConditionVisible(model, frameState) {
   return distance2 >= nearSquared && distance2 <= farSquared;
 }
 
+const scratchIBLReferenceFrameMatrix4 = new Matrix4();
+const scratchIBLReferenceFrameMatrix3 = new Matrix3();
 const scratchClippingPlanesMatrix = new Matrix4();
 
 /**
@@ -5576,7 +5579,32 @@ Model.prototype.update = function (frameState) {
     // models are part of a tileset these properties get transformed relative to
     // a common reference (such as the root).
     const referenceMatrix = defaultValue(this.referenceMatrix, modelMatrix);
-    this._imageBasedLighting.referenceMatrix = referenceMatrix;
+
+    if (
+      this._imageBasedLighting.useSphericalHarmonicCoefficients ||
+      this._imageBasedLighting.useSpecularEnvironmentMaps
+    ) {
+      let iblReferenceFrameMatrix3 = scratchIBLReferenceFrameMatrix3;
+      let iblReferenceFrameMatrix4 = scratchIBLReferenceFrameMatrix4;
+
+      iblReferenceFrameMatrix4 = Matrix4.multiply(
+        context.uniformState.view3D,
+        referenceMatrix,
+        iblReferenceFrameMatrix4
+      );
+      iblReferenceFrameMatrix3 = Matrix4.getMatrix3(
+        iblReferenceFrameMatrix4,
+        iblReferenceFrameMatrix3
+      );
+      iblReferenceFrameMatrix3 = Matrix3.getRotation(
+        iblReferenceFrameMatrix3,
+        iblReferenceFrameMatrix3
+      );
+      this._iblReferenceFrameMatrix = Matrix3.transpose(
+        iblReferenceFrameMatrix3,
+        this._iblReferenceFrameMatrix
+      );
+    }
 
     this._shouldRegenerateShaders =
       this._shouldRegenerateShaders ||
diff --git a/Source/Scene/ModelExperimental/ImageBasedLightingPipelineStage.js b/Source/Scene/ModelExperimental/ImageBasedLightingPipelineStage.js
index 1063009a3a44..1201ce0219f3 100644
--- a/Source/Scene/ModelExperimental/ImageBasedLightingPipelineStage.js
+++ b/Source/Scene/ModelExperimental/ImageBasedLightingPipelineStage.js
@@ -121,7 +121,7 @@ ImageBasedLightingPipelineStage.process = function (
       return imageBasedLighting.imageBasedLightingFactor;
     },
     model_iblReferenceFrameMatrix: function () {
-      return imageBasedLighting.iblReferenceFrameMatrix;
+      return model._iblReferenceFrameMatrix;
     },
     model_luminanceAtZenith: function () {
       return imageBasedLighting.luminanceAtZenith;
diff --git a/Source/Scene/ModelExperimental/ModelExperimental.js b/Source/Scene/ModelExperimental/ModelExperimental.js
index e48c1093b113..7d8fcd1beb9d 100644
--- a/Source/Scene/ModelExperimental/ModelExperimental.js
+++ b/Source/Scene/ModelExperimental/ModelExperimental.js
@@ -14,6 +14,7 @@ import ModelExperimentalUtility from "./ModelExperimentalUtility.js";
 import Pass from "../../Renderer/Pass.js";
 import Resource from "../../Core/Resource.js";
 import destroyObject from "../../Core/destroyObject.js";
+import Matrix3 from "../../Core/Matrix3.js";
 import Matrix4 from "../../Core/Matrix4.js";
 import ModelFeatureTable from "./ModelFeatureTable.js";
 import PointCloudShading from "../PointCloudShading.js";
@@ -146,6 +147,7 @@ export default function ModelExperimental(options) {
    * @private
    */
   this.referenceMatrix = undefined;
+  this._iblReferenceFrameMatrix = Matrix3.clone(Matrix3.IDENTITY); // Derived from reference matrix and the current view matrix
 
   this._resourcesLoaded = false;
   this._drawCommandsBuilt = false;
@@ -998,6 +1000,9 @@ ModelExperimental.prototype.resetDrawCommands = function () {
   this._drawCommandsBuilt = false;
 };
 
+const scratchIBLReferenceFrameMatrix4 = new Matrix4();
+const scratchIBLReferenceFrameMatrix3 = new Matrix3();
+
 /**
  * Called when {@link Viewer} or {@link CesiumWidget} render the scene to
  * get the draw commands needed to render this primitive.
@@ -1028,11 +1033,38 @@ ModelExperimental.prototype.update = function (frameState) {
     this._attenuation = this.pointCloudShading.attenuation;
   }
 
+  const context = frameState.context;
   const referenceMatrix = defaultValue(this.referenceMatrix, this.modelMatrix);
-  this._imageBasedLighting.referenceMatrix = referenceMatrix;
 
   // Update the image-based lighting for this model to detect any changes in parameters.
   this._imageBasedLighting.update(frameState);
+
+  if (
+    this._imageBasedLighting.useSphericalHarmonicCoefficients ||
+    this._imageBasedLighting.useSpecularEnvironmentMaps
+  ) {
+    let iblReferenceFrameMatrix3 = scratchIBLReferenceFrameMatrix3;
+    let iblReferenceFrameMatrix4 = scratchIBLReferenceFrameMatrix4;
+
+    iblReferenceFrameMatrix4 = Matrix4.multiply(
+      context.uniformState.view3D,
+      referenceMatrix,
+      iblReferenceFrameMatrix4
+    );
+    iblReferenceFrameMatrix3 = Matrix4.getMatrix3(
+      iblReferenceFrameMatrix4,
+      iblReferenceFrameMatrix3
+    );
+    iblReferenceFrameMatrix3 = Matrix3.getRotation(
+      iblReferenceFrameMatrix3,
+      iblReferenceFrameMatrix3
+    );
+    this._iblReferenceFrameMatrix = Matrix3.transpose(
+      iblReferenceFrameMatrix3,
+      this._iblReferenceFrameMatrix
+    );
+  }
+
   if (this._imageBasedLighting.shouldRegenerateShaders) {
     this.resetDrawCommands();
   }