diff --git a/lib/abr/simple_abr_manager.js b/lib/abr/simple_abr_manager.js
index ceeeeca91a..b252aed0d0 100644
--- a/lib/abr/simple_abr_manager.js
+++ b/lib/abr/simple_abr_manager.js
@@ -184,13 +184,30 @@ shaka.abr.SimpleAbrManager = class {
     if (this.resizeObserver_ && this.config_.restrictToElementSize) {
       const devicePixelRatio =
           this.config_.ignoreDevicePixelRatio ? 1 : window.devicePixelRatio;
-      maxHeight = this.mediaElement_.clientHeight * devicePixelRatio;
-      maxWidth = this.mediaElement_.clientWidth * devicePixelRatio;
+      maxHeight = Math.min(
+          maxHeight, this.mediaElement_.clientHeight * devicePixelRatio);
+      maxWidth = Math.min(
+          maxWidth, this.mediaElement_.clientWidth * devicePixelRatio);
     }
 
     // Get sorted Variants.
     let sortedVariants = SimpleAbrManager.filterAndSortVariants_(
-        this.config_.restrictions, this.variants_, maxHeight, maxWidth);
+        this.config_.restrictions, this.variants_,
+        /* maxHeight= */ Infinity, /* maxWidth= */ Infinity);
+
+    if (maxHeight != Infinity || maxWidth != Infinity) {
+      const resolutions = SimpleAbrManager.getResolutionList_(sortedVariants);
+      for (const resolution of resolutions) {
+        if (resolution.height >= maxHeight && resolution.width >= maxWidth) {
+          maxHeight = resolution.height;
+          maxWidth = resolution.width;
+          break;
+        }
+      }
+
+      sortedVariants = SimpleAbrManager.filterAndSortVariants_(
+          this.config_.restrictions, this.variants_, maxHeight, maxWidth);
+    }
 
     const defaultBandwidthEstimate = this.getDefaultBandwidth_();
     const currentBandwidth = this.bandwidthEstimator_.getBandwidthEstimate(
@@ -435,6 +452,29 @@ shaka.abr.SimpleAbrManager = class {
       return v1.bandwidth - v2.bandwidth;
     });
   }
+
+  /**
+   * @param {!Array.<shaka.extern.Variant>} variants
+   * @return {!Array.<{height: number, width: number}>}
+   * @private
+   */
+  static getResolutionList_(variants) {
+    const resolutions = [];
+    for (const variant of variants) {
+      const video = variant.video;
+      if (!video || !video.height || !video.width) {
+        continue;
+      }
+      resolutions.push({
+        height: video.height,
+        width: video.width,
+      });
+    }
+
+    return resolutions.sort((v1, v2) => {
+      return v1.width - v2.width;
+    });
+  }
 };