diff --git a/Core/ImageBuilder.cs b/Core/ImageBuilder.cs
index 8f73d8904..5366a1f77 100644
--- a/Core/ImageBuilder.cs
+++ b/Core/ImageBuilder.cs
@@ -1021,7 +1021,6 @@ protected override RequestedAction LayoutImage(ImageState s) {
                 //We first calculate the largest size the image can be under the width/height/maxwidth/maxheight restrictions.
                 //- pretending stretch=fill and scale=both
 
-                // TODO: why are doubles used here?  It just means lots of casts further down
                 //Temp vars - results stored in targetSize and areaSize
                 double width = s.settings.Width;
                 double height = s.settings.Height;
@@ -1058,20 +1057,29 @@ protected override RequestedAction LayoutImage(ImageState s) {
                 } else if (fit == FitMode.Crop) {
                     //We autocrop - so both target and area match the requested size
                     areaSize = targetSize = new SizeF((float)width, (float)height);
+                    
                     var minWidth = Math.Min(manualCropSize.Width, (float)width);
                     var minHeight = Math.Min(manualCropSize.Height, (float)height);
-                    if (scale == ScaleMode.DownscaleOnly 
-                        && ((manualCropSize.Width <= (float)width) != (manualCropSize.Height <= (float)height))) {
-                        // one of the dimensions is <= than the source (the other is not) and we're downscaling
-                        // TODO: what happens if mode=crop;scale=down but the target is larger than the source?
-                        areaSize = targetSize = new SizeF(minWidth, minHeight);
-                        manualCropRect = new RectangleF(0, 0, minWidth, minHeight);
-                        s.copyRect = PolygonMath.ToRectangle(PolygonMath.AlignWith(manualCropRect, s.copyRect, s.settings.Anchor));
-                    }
-                    else if (scale == ScaleMode.UpscaleCanvas) {
+
+                    bool cropWidthSmaller = manualCropSize.Width <= (float)width;
+                    bool cropHeightSmaller = manualCropSize.Height <= (float)height;
+
+                    // With both DownscaleOnly (where only one dimension is smaller than
+                    // requested) and UpscaleCanvas, we will have a targetSize based on the
+                    // minWidth & minHeight.
+                    // TODO: what happens if mode=crop;scale=down but the target is larger than the source?
+                    if ((scale == ScaleMode.DownscaleOnly && (cropWidthSmaller != cropHeightSmaller)) ||
+                        (scale == ScaleMode.UpscaleCanvas))
+                    {
                         targetSize = new SizeF(minWidth, minHeight);
                         manualCropRect = new RectangleF(0, 0, minWidth, minHeight);
                         s.copyRect = PolygonMath.ToRectangle(PolygonMath.AlignWith(manualCropRect, s.copyRect, s.settings.Anchor));
+
+                        // For DownscaleOnly, the areaSize is adjusted to the new targetSize as well.
+                        if (scale == ScaleMode.DownscaleOnly)
+                        {
+                            areaSize = targetSize;
+                        }
                     }
                     else
                     {