diff --git a/lib/player.js b/lib/player.js
index 88eca614c3..8b297ea19b 100644
--- a/lib/player.js
+++ b/lib/player.js
@@ -5686,6 +5686,10 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
       error.category != shaka.util.Error.Category.NETWORK) {
       return false;
     }
+    if (this.loadMode_ != shaka.Player.LoadMode.MEDIA_SOURCE) {
+      return false;
+    }
+    goog.asserts.assert(this.manifest_, 'Should have manifest!');
 
     if (!navigator.onLine) {
       // Don't restrict variants if we're completely offline, or else we end up