diff --git a/WebDriverAgent.xcodeproj/project.pbxproj b/WebDriverAgent.xcodeproj/project.pbxproj index 3630797a6..836267cce 100644 --- a/WebDriverAgent.xcodeproj/project.pbxproj +++ b/WebDriverAgent.xcodeproj/project.pbxproj @@ -38,11 +38,11 @@ 315A15072518CC2800A3A064 /* TouchSpotView.m in Sources */ = {isa = PBXBuildFile; fileRef = 315A15062518CC2800A3A064 /* TouchSpotView.m */; }; 315A150A2518D6F400A3A064 /* TouchViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = 315A15092518D6F400A3A064 /* TouchViewController.m */; }; 6385F4A7220A40760095BBDB /* XCUIApplicationProcessDelay.m in Sources */ = {isa = PBXBuildFile; fileRef = 6385F4A5220A40760095BBDB /* XCUIApplicationProcessDelay.m */; }; - 63CCF91221ECE4C700E94ABD /* FBImageIOScaler.h in Headers */ = {isa = PBXBuildFile; fileRef = 63CCF91021ECE4C700E94ABD /* FBImageIOScaler.h */; }; - 63CCF91321ECE4C700E94ABD /* FBImageIOScaler.m in Sources */ = {isa = PBXBuildFile; fileRef = 63CCF91121ECE4C700E94ABD /* FBImageIOScaler.m */; }; - 63FD950221F9D06100A3E356 /* FBImageIOScalerTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 631B523421F6174300625362 /* FBImageIOScalerTests.m */; }; - 63FD950321F9D06100A3E356 /* FBImageIOScalerTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 631B523421F6174300625362 /* FBImageIOScalerTests.m */; }; - 63FD950421F9D06200A3E356 /* FBImageIOScalerTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 631B523421F6174300625362 /* FBImageIOScalerTests.m */; }; + 63CCF91221ECE4C700E94ABD /* FBImageProcessor.h in Headers */ = {isa = PBXBuildFile; fileRef = 63CCF91021ECE4C700E94ABD /* FBImageProcessor.h */; }; + 63CCF91321ECE4C700E94ABD /* FBImageProcessor.m in Sources */ = {isa = PBXBuildFile; fileRef = 63CCF91121ECE4C700E94ABD /* FBImageProcessor.m */; }; + 63FD950221F9D06100A3E356 /* FBImageProcessorTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 631B523421F6174300625362 /* FBImageProcessorTests.m */; }; + 63FD950321F9D06100A3E356 /* FBImageProcessorTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 631B523421F6174300625362 /* FBImageProcessorTests.m */; }; + 63FD950421F9D06200A3E356 /* FBImageProcessorTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 631B523421F6174300625362 /* FBImageProcessorTests.m */; }; 641EE3452240C1C800173FCB /* UITestingUITests.m in Sources */ = {isa = PBXBuildFile; fileRef = EE9AB7FD1CAEE048008C271F /* UITestingUITests.m */; }; 641EE5D72240C5CA00173FCB /* FBScreenshotCommands.m in Sources */ = {isa = PBXBuildFile; fileRef = EE9AB75F1CAEDF0C008C271F /* FBScreenshotCommands.m */; }; 641EE5D92240C5CA00173FCB /* XCUIElement+FBPickerWheel.m in Sources */ = {isa = PBXBuildFile; fileRef = 7136A4781E8918E60024FC3D /* XCUIElement+FBPickerWheel.m */; }; @@ -80,7 +80,7 @@ 641EE5FE2240C5CA00173FCB /* XCUIElement+FBWebDriverAttributes.m in Sources */ = {isa = PBXBuildFile; fileRef = EEE376481D59FAE900ED88DD /* XCUIElement+FBWebDriverAttributes.m */; }; 641EE5FF2240C5CA00173FCB /* XCUIElement+FBForceTouch.m in Sources */ = {isa = PBXBuildFile; fileRef = EE8DDD7C20C5733B004D4925 /* XCUIElement+FBForceTouch.m */; }; 641EE6002240C5CA00173FCB /* FBTouchActionCommands.m in Sources */ = {isa = PBXBuildFile; fileRef = 71241D7A1FAE3D2500B9559F /* FBTouchActionCommands.m */; }; - 641EE6012240C5CA00173FCB /* FBImageIOScaler.m in Sources */ = {isa = PBXBuildFile; fileRef = 63CCF91121ECE4C700E94ABD /* FBImageIOScaler.m */; }; + 641EE6012240C5CA00173FCB /* FBImageProcessor.m in Sources */ = {isa = PBXBuildFile; fileRef = 63CCF91121ECE4C700E94ABD /* FBImageProcessor.m */; }; 641EE6022240C5CA00173FCB /* FBTouchIDCommands.m in Sources */ = {isa = PBXBuildFile; fileRef = EE9AB7631CAEDF0C008C271F /* FBTouchIDCommands.m */; }; 641EE6032240C5CA00173FCB /* FBDebugCommands.m in Sources */ = {isa = PBXBuildFile; fileRef = EE9AB7551CAEDF0C008C271F /* FBDebugCommands.m */; }; 641EE6042240C5CA00173FCB /* NSString+FBXMLSafeString.m in Sources */ = {isa = PBXBuildFile; fileRef = 716E0BCD1E917E810087A825 /* NSString+FBXMLSafeString.m */; }; @@ -222,7 +222,7 @@ 641EE6A32240C5CA00173FCB /* XCUIApplication+FBTouchAction.h in Headers */ = {isa = PBXBuildFile; fileRef = 71BD20711F86116100B36EC2 /* XCUIApplication+FBTouchAction.h */; }; 641EE6A42240C5CA00173FCB /* FBCommandHandler.h in Headers */ = {isa = PBXBuildFile; fileRef = EE9AB7751CAEDF0C008C271F /* FBCommandHandler.h */; settings = {ATTRIBUTES = (Public, ); }; }; 641EE6A52240C5CA00173FCB /* FBSessionCommands.h in Headers */ = {isa = PBXBuildFile; fileRef = EE9AB7601CAEDF0C008C271F /* FBSessionCommands.h */; }; - 641EE6A62240C5CA00173FCB /* FBImageIOScaler.h in Headers */ = {isa = PBXBuildFile; fileRef = 63CCF91021ECE4C700E94ABD /* FBImageIOScaler.h */; }; + 641EE6A62240C5CA00173FCB /* FBImageProcessor.h in Headers */ = {isa = PBXBuildFile; fileRef = 63CCF91021ECE4C700E94ABD /* FBImageProcessor.h */; }; 641EE6A72240C5CA00173FCB /* FBSession-Private.h in Headers */ = {isa = PBXBuildFile; fileRef = EE9AB7891CAEDF0C008C271F /* FBSession-Private.h */; }; 641EE6A82240C5CA00173FCB /* NSString+FBXMLSafeString.h in Headers */ = {isa = PBXBuildFile; fileRef = 716E0BCC1E917E810087A825 /* NSString+FBXMLSafeString.h */; }; 641EE6A92240C5CA00173FCB /* FBCommandStatus.h in Headers */ = {isa = PBXBuildFile; fileRef = EE9AB7761CAEDF0C008C271F /* FBCommandStatus.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -918,11 +918,11 @@ 315A15082518D6F400A3A064 /* TouchViewController.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = TouchViewController.h; sourceTree = ""; }; 315A15092518D6F400A3A064 /* TouchViewController.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = TouchViewController.m; sourceTree = ""; }; 44757A831D42CE8300ECF35E /* XCUIDeviceRotationTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = XCUIDeviceRotationTests.m; sourceTree = ""; }; - 631B523421F6174300625362 /* FBImageIOScalerTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FBImageIOScalerTests.m; sourceTree = ""; }; + 631B523421F6174300625362 /* FBImageProcessorTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FBImageProcessorTests.m; sourceTree = ""; }; 633E904A220DEE7F007CADF9 /* XCUIApplicationProcessDelay.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = XCUIApplicationProcessDelay.h; sourceTree = ""; }; 6385F4A5220A40760095BBDB /* XCUIApplicationProcessDelay.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = XCUIApplicationProcessDelay.m; sourceTree = ""; }; - 63CCF91021ECE4C700E94ABD /* FBImageIOScaler.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FBImageIOScaler.h; sourceTree = ""; }; - 63CCF91121ECE4C700E94ABD /* FBImageIOScaler.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FBImageIOScaler.m; sourceTree = ""; }; + 63CCF91021ECE4C700E94ABD /* FBImageProcessor.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FBImageProcessor.h; sourceTree = ""; }; + 63CCF91121ECE4C700E94ABD /* FBImageProcessor.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FBImageProcessor.m; sourceTree = ""; }; 641EE2DA2240BBE300173FCB /* WebDriverAgentRunner_tvOS.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = WebDriverAgentRunner_tvOS.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 641EE6F82240C5CA00173FCB /* WebDriverAgentLib_tvOS.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = WebDriverAgentLib_tvOS.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 641EE7042240CDCF00173FCB /* XCUIElement+FBTVFocuse.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "XCUIElement+FBTVFocuse.h"; sourceTree = ""; }; @@ -1890,8 +1890,8 @@ EE3A18611CDE618F00DE4205 /* FBErrorBuilder.m */, EE6A89381D0B38640083E92B /* FBFailureProofTestCase.h */, EE6A89391D0B38640083E92B /* FBFailureProofTestCase.m */, - 63CCF91021ECE4C700E94ABD /* FBImageIOScaler.h */, - 63CCF91121ECE4C700E94ABD /* FBImageIOScaler.m */, + 63CCF91021ECE4C700E94ABD /* FBImageProcessor.h */, + 63CCF91121ECE4C700E94ABD /* FBImageProcessor.m */, 7150348521A6DAD600A0F4BA /* FBImageUtils.h */, 7150348621A6DAD600A0F4BA /* FBImageUtils.m */, EE9B76A31CF7A43900275851 /* FBLogger.h */, @@ -2004,7 +2004,7 @@ 71E504941DF59BAD0020C32A /* XCUIElementAttributesTests.m */, EEBBD48D1D4785FC00656A81 /* XCUIElementFBFindTests.m */, EE1E06E11D181CC9007CF043 /* XCUIElementHelperIntegrationTests.m */, - 631B523421F6174300625362 /* FBImageIOScalerTests.m */, + 631B523421F6174300625362 /* FBImageProcessorTests.m */, 644D9CCD230E1F1A00C90459 /* FBConfigurationTests.m */, ); path = IntegrationTests; @@ -2381,7 +2381,7 @@ 641EE6A42240C5CA00173FCB /* FBCommandHandler.h in Headers */, 641EE6A52240C5CA00173FCB /* FBSessionCommands.h in Headers */, 641EE70C2240CE2D00173FCB /* FBTVNavigationTracker.h in Headers */, - 641EE6A62240C5CA00173FCB /* FBImageIOScaler.h in Headers */, + 641EE6A62240C5CA00173FCB /* FBImageProcessor.h in Headers */, 641EE6A72240C5CA00173FCB /* FBSession-Private.h in Headers */, 641EE6A82240C5CA00173FCB /* NSString+FBXMLSafeString.h in Headers */, 64E3502F2AC0B6FE005F3ACB /* NSDictionary+FBUtf8SafeDictionary.h in Headers */, @@ -2622,7 +2622,7 @@ EE158AC81CBD456F00A3E3F0 /* FBSessionCommands.h in Headers */, 71C8E55125399A6B008572C1 /* XCUIApplication+FBQuiescence.h in Headers */, 641EE70B2240CE2D00173FCB /* FBTVNavigationTracker.h in Headers */, - 63CCF91221ECE4C700E94ABD /* FBImageIOScaler.h in Headers */, + 63CCF91221ECE4C700E94ABD /* FBImageProcessor.h in Headers */, EE158AE31CBD456F00A3E3F0 /* FBSession-Private.h in Headers */, 716E0BCE1E917E810087A825 /* NSString+FBXMLSafeString.h in Headers */, EE158ACF1CBD456F00A3E3F0 /* FBCommandStatus.h in Headers */, @@ -3122,7 +3122,7 @@ 641EE6002240C5CA00173FCB /* FBTouchActionCommands.m in Sources */, 719DCF182601EAFB000E765F /* FBNotificationsHelper.m in Sources */, 714EAA102673FDFE005C5B47 /* FBCapabilities.m in Sources */, - 641EE6012240C5CA00173FCB /* FBImageIOScaler.m in Sources */, + 641EE6012240C5CA00173FCB /* FBImageProcessor.m in Sources */, 641EE6022240C5CA00173FCB /* FBTouchIDCommands.m in Sources */, 641EE6032240C5CA00173FCB /* FBDebugCommands.m in Sources */, 641EE6042240C5CA00173FCB /* NSString+FBXMLSafeString.m in Sources */, @@ -3236,7 +3236,7 @@ EEE3764A1D59FAE900ED88DD /* XCUIElement+FBWebDriverAttributes.m in Sources */, EE8DDD7E20C5733C004D4925 /* XCUIElement+FBForceTouch.m in Sources */, 71241D7C1FAE3D2500B9559F /* FBTouchActionCommands.m in Sources */, - 63CCF91321ECE4C700E94ABD /* FBImageIOScaler.m in Sources */, + 63CCF91321ECE4C700E94ABD /* FBImageProcessor.m in Sources */, EE158ACB1CBD456F00A3E3F0 /* FBTouchIDCommands.m in Sources */, 71F5BE51252F14EB00EE9EBA /* FBExceptions.m in Sources */, EE158ABD1CBD456F00A3E3F0 /* FBDebugCommands.m in Sources */, @@ -3307,7 +3307,7 @@ files = ( 71241D801FAF087500B9559F /* FBW3CMultiTouchActionsIntegrationTests.m in Sources */, 71241D7E1FAF084E00B9559F /* FBW3CTouchActionsIntegrationTests.m in Sources */, - 63FD950221F9D06100A3E356 /* FBImageIOScalerTests.m in Sources */, + 63FD950221F9D06100A3E356 /* FBImageProcessorTests.m in Sources */, 719CD8FF2126C90200C7D0C2 /* FBAutoAlertsHandlerTests.m in Sources */, EE2202131ECC612200A29571 /* FBIntegrationTestCase.m in Sources */, 71BD20781F869E0F00B36EC2 /* FBAppiumTouchActionsIntegrationTests.m in Sources */, @@ -3324,7 +3324,7 @@ buildActionMask = 2147483647; files = ( EE5095E51EBCC9090028E2FE /* FBTypingTest.m in Sources */, - 63FD950321F9D06100A3E356 /* FBImageIOScalerTests.m in Sources */, + 63FD950321F9D06100A3E356 /* FBImageProcessorTests.m in Sources */, EE5095EB1EBCC9090028E2FE /* XCElementSnapshotHitPointTests.m in Sources */, EE5095EC1EBCC9090028E2FE /* XCUIApplicationHelperTests.m in Sources */, 7136C0F9243A182400921C76 /* FBW3CTypeActionsTests.m in Sources */, @@ -3397,7 +3397,7 @@ 7119E1EC1E891F8600D0B125 /* FBPickerWheelSelectTests.m in Sources */, 71ACF5B8242F2FDC00F0AAD4 /* FBSafariAlertTests.m in Sources */, EE1E06DA1D1808C2007CF043 /* FBIntegrationTestCase.m in Sources */, - 63FD950421F9D06200A3E356 /* FBImageIOScalerTests.m in Sources */, + 63FD950421F9D06200A3E356 /* FBImageProcessorTests.m in Sources */, EE05BAFA1D13003C00A3EB00 /* FBKeyboardTests.m in Sources */, EE55B3271D1D54CF003AAAEC /* FBScrollingTests.m in Sources */, EE6A89371D0B35920083E92B /* FBFailureProofTestCaseTests.m in Sources */, diff --git a/WebDriverAgentLib/Commands/FBSessionCommands.m b/WebDriverAgentLib/Commands/FBSessionCommands.m index 7bd87a768..4a7ae1a8d 100644 --- a/WebDriverAgentLib/Commands/FBSessionCommands.m +++ b/WebDriverAgentLib/Commands/FBSessionCommands.m @@ -280,6 +280,7 @@ + (NSArray *)routes FB_SETTING_MJPEG_SERVER_SCREENSHOT_QUALITY: @([FBConfiguration mjpegServerScreenshotQuality]), FB_SETTING_MJPEG_SERVER_FRAMERATE: @([FBConfiguration mjpegServerFramerate]), FB_SETTING_MJPEG_SCALING_FACTOR: @([FBConfiguration mjpegScalingFactor]), + FB_SETTING_MJPEG_FIX_ORIENTATION: @([FBConfiguration mjpegShouldFixOrientation]), FB_SETTING_SCREENSHOT_QUALITY: @([FBConfiguration screenshotQuality]), FB_SETTING_KEYBOARD_AUTOCORRECTION: @([FBConfiguration keyboardAutocorrection]), FB_SETTING_KEYBOARD_PREDICTION: @([FBConfiguration keyboardPrediction]), @@ -327,6 +328,9 @@ + (NSArray *)routes if (nil != [settings objectForKey:FB_SETTING_MJPEG_SCALING_FACTOR]) { [FBConfiguration setMjpegScalingFactor:[[settings objectForKey:FB_SETTING_MJPEG_SCALING_FACTOR] unsignedIntegerValue]]; } + if (nil != [settings objectForKey:FB_SETTING_MJPEG_FIX_ORIENTATION]) { + [FBConfiguration setMjpegShouldFixOrientation:[[settings objectForKey:FB_SETTING_MJPEG_FIX_ORIENTATION] boolValue]]; + } if (nil != [settings objectForKey:FB_SETTING_KEYBOARD_AUTOCORRECTION]) { [FBConfiguration setKeyboardAutocorrection:[[settings objectForKey:FB_SETTING_KEYBOARD_AUTOCORRECTION] boolValue]]; } diff --git a/WebDriverAgentLib/Utilities/FBConfiguration.h b/WebDriverAgentLib/Utilities/FBConfiguration.h index 8bb9d48ca..dd0015e56 100644 --- a/WebDriverAgentLib/Utilities/FBConfiguration.h +++ b/WebDriverAgentLib/Utilities/FBConfiguration.h @@ -84,6 +84,16 @@ extern NSString *const FBSnapshotMaxDepthKey; + (NSUInteger)mjpegServerScreenshotQuality; + (void)setMjpegServerScreenshotQuality:(NSUInteger)quality; +/** + Whether to apply orientation fixes to the streamed JPEG images. + This is an expensive operation and it is disabled by default, so screenshots + are returned in portrait, but their actual orientation value could still be found in the EXIF + metadata. + ! Enablement of this setting may lead to WDA process termination because of an excessive CPU usage. + */ ++ (BOOL)mjpegShouldFixOrientation; ++ (void)setMjpegShouldFixOrientation:(BOOL)enabled; + /** The framerate at which the background screenshots broadcaster should broadcast screenshots in range 1..60. The default value is 10 (Frames Per Second). @@ -93,7 +103,7 @@ extern NSString *const FBSnapshotMaxDepthKey; + (void)setMjpegServerFramerate:(NSUInteger)framerate; /** - The quality of display screenshots. The higher quality you set is the bigger screenshot size is. + The quality of display screenshots. The higher quality you set is the bigger screenshot size is. The highest quality value is 0 (lossless PNG) or 3 (lossless HEIC). The lowest quality is 2 (highly compressed JPEG). The default quality value is 3 (lossless HEIC). See https://developer.apple.com/documentation/xctest/xctimagequality?language=objc @@ -112,7 +122,10 @@ extern NSString *const FBSnapshotMaxDepthKey; + (NSInteger)mjpegServerPort; /** - The scaling factor for frames of the mjpeg stream (Default values is 100 and does not perform scaling). + The scaling factor for frames of the mjpeg stream. The default (and maximum) value is 100, + which does not perform any scaling. The minimum value must be greater than zero. + ! Setting this to a value less than 100, especially together with orientation fixing enabled + ! may lead to WDA process termination because of an excessive CPU usage. */ + (NSUInteger)mjpegScalingFactor; + (void)setMjpegScalingFactor:(NSUInteger)scalingFactor; diff --git a/WebDriverAgentLib/Utilities/FBConfiguration.m b/WebDriverAgentLib/Utilities/FBConfiguration.m index f149d2193..03b6a6b1b 100644 --- a/WebDriverAgentLib/Utilities/FBConfiguration.m +++ b/WebDriverAgentLib/Utilities/FBConfiguration.m @@ -33,6 +33,7 @@ static BOOL FBShouldUseSingletonTestManager = YES; static NSUInteger FBMjpegScalingFactor = 100; +static BOOL FBMjpegShouldFixOrientation = NO; static NSUInteger FBMjpegServerScreenshotQuality = 25; static NSUInteger FBMjpegServerFramerate = 10; @@ -143,6 +144,15 @@ + (void)setMjpegScalingFactor:(NSUInteger)scalingFactor { FBMjpegScalingFactor = scalingFactor; } ++ (BOOL)mjpegShouldFixOrientation +{ + return FBMjpegShouldFixOrientation; +} + ++ (void)setMjpegShouldFixOrientation:(BOOL)enabled { + FBMjpegShouldFixOrientation = enabled; +} + + (BOOL)verboseLoggingEnabled { return [NSProcessInfo.processInfo.environment[@"VERBOSE_LOGGING"] boolValue]; diff --git a/WebDriverAgentLib/Utilities/FBImageIOScaler.m b/WebDriverAgentLib/Utilities/FBImageIOScaler.m deleted file mode 100644 index 0ca87b747..000000000 --- a/WebDriverAgentLib/Utilities/FBImageIOScaler.m +++ /dev/null @@ -1,195 +0,0 @@ -/** - * Copyright (c) 2015-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ - -#import "FBImageIOScaler.h" - -#import -#import -@import UniformTypeIdentifiers; - -#import "FBConfiguration.h" -#import "FBErrorBuilder.h" -#import "FBLogger.h" - -const CGFloat FBMinScalingFactor = 0.01f; -const CGFloat FBMaxScalingFactor = 1.0f; -const CGFloat FBMinCompressionQuality = 0.0f; -const CGFloat FBMaxCompressionQuality = 1.0f; - -@interface FBImageIOScaler () - -@property (nonatomic) NSData *nextImage; -@property (nonatomic, readonly) NSLock *nextImageLock; -@property (nonatomic, readonly) dispatch_queue_t scalingQueue; - -@end - -@implementation FBImageIOScaler - -- (id)init -{ - self = [super init]; - if (self) { - _nextImageLock = [[NSLock alloc] init]; - _scalingQueue = dispatch_queue_create("image.scaling.queue", NULL); - } - return self; -} - -- (void)submitImage:(NSData *)image - scalingFactor:(CGFloat)scalingFactor - compressionQuality:(CGFloat)compressionQuality - completionHandler:(void (^)(NSData *))completionHandler -{ - [self.nextImageLock lock]; - if (self.nextImage != nil) { - [FBLogger verboseLog:@"Discarding screenshot"]; - } - scalingFactor = MAX(FBMinScalingFactor, MIN(FBMaxScalingFactor, scalingFactor)); - compressionQuality = MAX(FBMinCompressionQuality, MIN(FBMaxCompressionQuality, compressionQuality)); - self.nextImage = image; - [self.nextImageLock unlock]; - -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wcompletion-handler" - dispatch_async(self.scalingQueue, ^{ - [self.nextImageLock lock]; - NSData *next = self.nextImage; - self.nextImage = nil; - [self.nextImageLock unlock]; - if (next == nil) { - return; - } - - NSError *error; - NSData *scaled = [self scaledJpegImageWithImage:next - scalingFactor:scalingFactor - compressionQuality:compressionQuality - error:&error]; - if (scaled == nil) { - [FBLogger logFmt:@"%@", error.description]; - return; - } - completionHandler(scaled); - }); -#pragma clang diagnostic pop -} - -// This method is more optimized for JPEG scaling -// and should be used in `submitImage` API, while the `scaledImageWithImage` -// one is more generic -- (nullable NSData *)scaledJpegImageWithImage:(NSData *)image - scalingFactor:(CGFloat)scalingFactor - compressionQuality:(CGFloat)compressionQuality - error:(NSError **)error -{ - CGImageSourceRef imageData = CGImageSourceCreateWithData((CFDataRef)image, nil); - CGSize size = [self.class imageSizeWithImage:imageData]; - CGFloat scaledMaxPixelSize = MAX(size.width, size.height) * scalingFactor; - CFDictionaryRef params = (__bridge CFDictionaryRef)@{ - (const NSString *)kCGImageSourceCreateThumbnailWithTransform: @(YES), - (const NSString *)kCGImageSourceCreateThumbnailFromImageIfAbsent: @(YES), - (const NSString *)kCGImageSourceThumbnailMaxPixelSize: @(scaledMaxPixelSize) - }; - CGImageRef scaled = CGImageSourceCreateThumbnailAtIndex(imageData, 0, params); - CFRelease(imageData); - if (nil == scaled) { - [[[FBErrorBuilder builder] - withDescriptionFormat:@"Failed to scale the image"] - buildError:error]; - return nil; - } - NSData *resData = [self jpegDataWithImage:scaled - compressionQuality:compressionQuality]; - if (nil == resData) { - [[[FBErrorBuilder builder] - withDescriptionFormat:@"Failed to compress the image to JPEG format"] - buildError:error]; - } - CGImageRelease(scaled); - return resData; -} - -- (nullable NSData *)scaledImageWithImage:(NSData *)image - uti:(UTType *)uti - rect:(CGRect)rect - scalingFactor:(CGFloat)scalingFactor - compressionQuality:(CGFloat)compressionQuality - error:(NSError **)error -{ - UIImage *uiImage = [UIImage imageWithData:image]; - CGSize size = uiImage.size; - CGSize scaledSize = CGSizeMake(size.width * scalingFactor, size.height * scalingFactor); - UIGraphicsBeginImageContext(scaledSize); - UIImageOrientation orientation = uiImage.imageOrientation; -#if !TARGET_OS_TV - if (FBConfiguration.screenshotOrientation == UIInterfaceOrientationPortrait) { - orientation = UIImageOrientationUp; - } else if (FBConfiguration.screenshotOrientation == UIInterfaceOrientationPortraitUpsideDown) { - orientation = UIImageOrientationDown; - } else if (FBConfiguration.screenshotOrientation == UIInterfaceOrientationLandscapeLeft) { - orientation = UIImageOrientationRight; - } else if (FBConfiguration.screenshotOrientation == UIInterfaceOrientationLandscapeRight) { - orientation = UIImageOrientationLeft; - } -#endif - uiImage = [UIImage imageWithCGImage:(CGImageRef)uiImage.CGImage - scale:uiImage.scale - orientation:orientation]; - [uiImage drawInRect:CGRectMake(0, 0, scaledSize.width, scaledSize.height)]; - UIImage *resultImage = UIGraphicsGetImageFromCurrentImageContext(); - UIGraphicsEndImageContext(); - - if (!CGRectIsNull(rect)) { - UIGraphicsBeginImageContext(rect.size); - [resultImage drawAtPoint:CGPointMake(-rect.origin.x, -rect.origin.y)]; - resultImage = UIGraphicsGetImageFromCurrentImageContext(); - UIGraphicsEndImageContext(); - } - - return [uti conformsToType:UTTypePNG] - ? UIImagePNGRepresentation(resultImage) - : UIImageJPEGRepresentation(resultImage, compressionQuality); -} - -- (nullable NSData *)jpegDataWithImage:(CGImageRef)imageRef - compressionQuality:(CGFloat)compressionQuality -{ - NSMutableData *newImageData = [NSMutableData data]; - CGImageDestinationRef imageDestination = CGImageDestinationCreateWithData( - (__bridge CFMutableDataRef) newImageData, - (__bridge CFStringRef) UTTypeJPEG.identifier, - 1, - NULL); - CFDictionaryRef compressionOptions = (__bridge CFDictionaryRef)@{ - (const NSString *)kCGImageDestinationLossyCompressionQuality: @(compressionQuality) - }; - CGImageDestinationAddImage(imageDestination, imageRef, compressionOptions); - if(!CGImageDestinationFinalize(imageDestination)) { - [FBLogger log:@"Failed to write the image"]; - newImageData = nil; - } - CFRelease(imageDestination); - return newImageData; -} - -+ (CGSize)imageSizeWithImage:(CGImageSourceRef)imageSource -{ - NSDictionary *options = @{ - (const NSString *)kCGImageSourceShouldCache: @(NO) - }; - CFDictionaryRef properties = CGImageSourceCopyPropertiesAtIndex(imageSource, 0, (CFDictionaryRef)options); - NSNumber *width = [(__bridge NSDictionary *)properties objectForKey:(const NSString *)kCGImagePropertyPixelWidth]; - NSNumber *height = [(__bridge NSDictionary *)properties objectForKey:(const NSString *)kCGImagePropertyPixelHeight]; - CGSize size = CGSizeMake([width floatValue], [height floatValue]); - CFRelease(properties); - return size; -} - -@end diff --git a/WebDriverAgentLib/Utilities/FBImageIOScaler.h b/WebDriverAgentLib/Utilities/FBImageProcessor.h similarity index 64% rename from WebDriverAgentLib/Utilities/FBImageIOScaler.h rename to WebDriverAgentLib/Utilities/FBImageProcessor.h index 476daba04..a350bd9ce 100644 --- a/WebDriverAgentLib/Utilities/FBImageIOScaler.h +++ b/WebDriverAgentLib/Utilities/FBImageProcessor.h @@ -20,7 +20,7 @@ extern const CGFloat FBMaxScalingFactor; extern const CGFloat FBMinCompressionQuality; extern const CGFloat FBMaxCompressionQuality; -@interface FBImageIOScaler : NSObject +@interface FBImageProcessor : NSObject /** Puts the passed image on the queue and dispatches a scaling operation. If there is already a image on the @@ -29,34 +29,27 @@ extern const CGFloat FBMaxCompressionQuality; @param image The image to scale down @param completionHandler called after successfully scaling down an image @param scalingFactor the scaling factor in range 0.01..1.0. A value of 1.0 won't perform scaling at all - @param compressionQuality the compression quality in range 0.0..1.0 (0.0 for max. compression and 1.0 for lossless compression) - Only applicable for UTTypeJPEG */ -- (void)submitImage:(NSData *)image - scalingFactor:(CGFloat)scalingFactor - compressionQuality:(CGFloat)compressionQuality - completionHandler:(void (^)(NSData *))completionHandler; +- (void)submitImageData:(NSData *)image + scalingFactor:(CGFloat)scalingFactor + completionHandler:(void (^)(NSData *))completionHandler; /** Scales and crops the source image @param image The source image data @param uti Either UTTypePNG or UTTypeJPEG - @param rect The cropping rectange for the screenshot. The value is expected to be non-scaled one - since it happens after scaling/orientation change. - CGRectNull could be used to take a screenshot of the full screen. @param scalingFactor Scaling factor in range 0.01..1.0. A value of 1.0 won't perform scaling at all @param compressionQuality the compression quality in range 0.0..1.0 (0.0 for max. compression and 1.0 for lossless compression). Only works if UTI is set to kUTTypeJPEG @param error The actual error instance if the returned result is nil @returns Processed image data compressed according to the given UTI or nil in case of a failure */ -- (nullable NSData *)scaledImageWithImage:(NSData *)image - uti:(UTType *)uti - rect:(CGRect)rect - scalingFactor:(CGFloat)scalingFactor - compressionQuality:(CGFloat)compressionQuality - error:(NSError **)error; +- (nullable NSData *)scaledImageWithData:(NSData *)image + uti:(UTType *)uti + scalingFactor:(CGFloat)scalingFactor + compressionQuality:(CGFloat)compressionQuality + error:(NSError **)error; @end diff --git a/WebDriverAgentLib/Utilities/FBImageProcessor.m b/WebDriverAgentLib/Utilities/FBImageProcessor.m new file mode 100644 index 000000000..16601ce8e --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBImageProcessor.m @@ -0,0 +1,171 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#import "FBImageProcessor.h" + +#import +#import +@import UniformTypeIdentifiers; + +#import "FBConfiguration.h" +#import "FBErrorBuilder.h" +#import "FBImageUtils.h" +#import "FBLogger.h" + +const CGFloat FBMinScalingFactor = 0.01f; +const CGFloat FBMaxScalingFactor = 1.0f; +const CGFloat FBMinCompressionQuality = 0.0f; +const CGFloat FBMaxCompressionQuality = 1.0f; + +@interface FBImageProcessor () + +@property (nonatomic) NSData *nextImage; +@property (nonatomic, readonly) NSLock *nextImageLock; +@property (nonatomic, readonly) dispatch_queue_t scalingQueue; + +@end + +@implementation FBImageProcessor + +- (id)init +{ + self = [super init]; + if (self) { + _nextImageLock = [[NSLock alloc] init]; + _scalingQueue = dispatch_queue_create("image.scaling.queue", NULL); + } + return self; +} + +- (void)submitImageData:(NSData *)image + scalingFactor:(CGFloat)scalingFactor + completionHandler:(void (^)(NSData *))completionHandler +{ + [self.nextImageLock lock]; + if (self.nextImage != nil) { + [FBLogger verboseLog:@"Discarding screenshot"]; + } + self.nextImage = image; + [self.nextImageLock unlock]; + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcompletion-handler" + dispatch_async(self.scalingQueue, ^{ + [self.nextImageLock lock]; + NSData *nextImageData = self.nextImage; + self.nextImage = nil; + [self.nextImageLock unlock]; + if (nextImageData == nil) { + return; + } + + // We do not want this value to be too high because then we get images larger in size than original ones + // Although, we also don't want to lose too much of the quality on recompression + CGFloat recompressionQuality = MAX(0.9, + MIN(FBMaxCompressionQuality, FBConfiguration.mjpegServerScreenshotQuality / 100.0)); + NSData *thumbnailData = [self.class fixedImageDataWithImageData:nextImageData + scalingFactor:scalingFactor + uti:UTTypeJPEG + compressionQuality:recompressionQuality + // iOS always returns screnshots in portrait orientation, but puts the real value into the metadata + // Use it with care. See https://github.com/appium/WebDriverAgent/pull/812 + fixOrientation:FBConfiguration.mjpegShouldFixOrientation + desiredOrientation:nil]; + completionHandler(thumbnailData ?: nextImageData); + }); +#pragma clang diagnostic pop +} + ++ (nullable NSData *)fixedImageDataWithImageData:(NSData *)imageData + scalingFactor:(CGFloat)scalingFactor + uti:(UTType *)uti + compressionQuality:(CGFloat)compressionQuality + fixOrientation:(BOOL)fixOrientation + desiredOrientation:(nullable NSNumber *)orientation +{ + scalingFactor = MAX(FBMinScalingFactor, MIN(FBMaxScalingFactor, scalingFactor)); + BOOL usesScaling = scalingFactor > 0.0 && scalingFactor < FBMaxScalingFactor; + @autoreleasepool { + if (!usesScaling && !fixOrientation) { + return [uti conformsToType:UTTypePNG] ? FBToPngData(imageData) : FBToJpegData(imageData, compressionQuality); + } + + UIImage *image = [UIImage imageWithData:imageData]; + if (nil == image + || ((image.imageOrientation == UIImageOrientationUp || !fixOrientation) && !usesScaling)) { + return [uti conformsToType:UTTypePNG] ? FBToPngData(imageData) : FBToJpegData(imageData, compressionQuality); + } + + CGSize scaledSize = CGSizeMake(image.size.width * scalingFactor, image.size.height * scalingFactor); + if (!fixOrientation && usesScaling) { + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + __block UIImage *result = nil; + [image prepareThumbnailOfSize:scaledSize + completionHandler:^(UIImage * _Nullable thumbnail) { + result = thumbnail; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (nil == result) { + return [uti conformsToType:UTTypePNG] ? FBToPngData(imageData) : FBToJpegData(imageData, compressionQuality); + } + return [uti conformsToType:UTTypePNG] + ? UIImagePNGRepresentation(result) + : UIImageJPEGRepresentation(result, compressionQuality); + } + + UIGraphicsImageRendererFormat *format = [[UIGraphicsImageRendererFormat alloc] init]; + format.scale = scalingFactor; + UIGraphicsImageRenderer *renderer = [[UIGraphicsImageRenderer alloc] initWithSize:scaledSize + format:format]; + UIImageOrientation desiredOrientation = orientation == nil + ? image.imageOrientation + : (UIImageOrientation)orientation.integerValue; + UIImage *uiImage = [UIImage imageWithCGImage:(CGImageRef)image.CGImage + scale:image.scale + orientation:desiredOrientation]; + return [uti conformsToType:UTTypePNG] + ? [renderer PNGDataWithActions:^(UIGraphicsImageRendererContext * _Nonnull rendererContext) { + [uiImage drawInRect:CGRectMake(0, 0, scaledSize.width, scaledSize.height)]; + }] + : [renderer JPEGDataWithCompressionQuality:compressionQuality + actions:^(UIGraphicsImageRendererContext * _Nonnull rendererContext) { + [uiImage drawInRect:CGRectMake(0, 0, scaledSize.width, scaledSize.height)]; + }]; + } +} + +- (nullable NSData *)scaledImageWithData:(NSData *)imageData + uti:(UTType *)uti + scalingFactor:(CGFloat)scalingFactor + compressionQuality:(CGFloat)compressionQuality + error:(NSError **)error +{ + NSNumber *orientation = nil; +#if !TARGET_OS_TV + if (FBConfiguration.screenshotOrientation == UIInterfaceOrientationPortrait) { + orientation = @(UIImageOrientationUp); + } else if (FBConfiguration.screenshotOrientation == UIInterfaceOrientationPortraitUpsideDown) { + orientation = @(UIImageOrientationDown); + } else if (FBConfiguration.screenshotOrientation == UIInterfaceOrientationLandscapeLeft) { + orientation = @(UIImageOrientationRight); + } else if (FBConfiguration.screenshotOrientation == UIInterfaceOrientationLandscapeRight) { + orientation = @(UIImageOrientationLeft); + } +#endif + NSData *resultData = [self.class fixedImageDataWithImageData:imageData + scalingFactor:scalingFactor + uti:uti + compressionQuality:compressionQuality + fixOrientation:YES + desiredOrientation:orientation]; + return resultData ?: imageData; +} + +@end diff --git a/WebDriverAgentLib/Utilities/FBImageUtils.h b/WebDriverAgentLib/Utilities/FBImageUtils.h index 96ea15a6b..15a04f6fe 100644 --- a/WebDriverAgentLib/Utilities/FBImageUtils.h +++ b/WebDriverAgentLib/Utilities/FBImageUtils.h @@ -17,4 +17,10 @@ BOOL FBIsPngImage(NSData *imageData); /*! Converts the given image data to a PNG representation if necessary */ NSData *_Nullable FBToPngData(NSData *imageData); +/*! Returns YES if the data contains a JPG image */ +BOOL FBIsJpegImage(NSData *imageData); + +/*! Converts the given image data to a JPG representation if necessary */ +NSData *_Nullable FBToJpegData(NSData *imageData, CGFloat compressionQuality); + NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBImageUtils.m b/WebDriverAgentLib/Utilities/FBImageUtils.m index 5983ddc30..b1d2f9b5b 100644 --- a/WebDriverAgentLib/Utilities/FBImageUtils.m +++ b/WebDriverAgentLib/Utilities/FBImageUtils.m @@ -12,8 +12,11 @@ #import "FBMacros.h" #import "FBConfiguration.h" +// https://en.wikipedia.org/wiki/List_of_file_signatures static uint8_t PNG_MAGIC[] = { 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A }; static const NSUInteger PNG_MAGIC_LEN = 8; +static uint8_t JPG_MAGIC[] = { 0xff, 0xd8, 0xff }; +static const NSUInteger JPG_MAGIC_LEN = 3; BOOL FBIsPngImage(NSData *imageData) { @@ -45,3 +48,34 @@ BOOL FBIsPngImage(NSData *imageData) UIImage *image = [UIImage imageWithData:imageData]; return nil == image ? nil : (NSData *)UIImagePNGRepresentation(image); } + +BOOL FBIsJpegImage(NSData *imageData) +{ + if (nil == imageData || [imageData length] < JPG_MAGIC_LEN) { + return NO; + } + + static NSData* jpgMagicStartData = nil; + static dispatch_once_t onceJpgToken; + dispatch_once(&onceJpgToken, ^{ + jpgMagicStartData = [NSData dataWithBytesNoCopy:(void*)JPG_MAGIC length:JPG_MAGIC_LEN freeWhenDone:NO]; + }); + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wassign-enum" + NSRange range = [imageData rangeOfData:jpgMagicStartData options:kNilOptions range:NSMakeRange(0, JPG_MAGIC_LEN)]; +#pragma clang diagnostic pop + return range.location != NSNotFound; +} + +NSData *FBToJpegData(NSData *imageData, CGFloat compressionQuality) { + if (nil == imageData || [imageData length] < JPG_MAGIC_LEN) { + return nil; + } + if (FBIsJpegImage(imageData)) { + return imageData; + } + + UIImage *image = [UIImage imageWithData:imageData]; + return nil == image ? nil : (NSData *)UIImageJPEGRepresentation(image, compressionQuality); +} diff --git a/WebDriverAgentLib/Utilities/FBMjpegServer.m b/WebDriverAgentLib/Utilities/FBMjpegServer.m index 51fa48687..a0970e256 100644 --- a/WebDriverAgentLib/Utilities/FBMjpegServer.m +++ b/WebDriverAgentLib/Utilities/FBMjpegServer.m @@ -17,7 +17,8 @@ #import "FBConfiguration.h" #import "FBLogger.h" #import "FBScreenshot.h" -#import "FBImageIOScaler.h" +#import "FBImageProcessor.h" +#import "FBImageUtils.h" #import "XCUIScreen.h" static const NSUInteger MAX_FPS = 60; @@ -31,7 +32,7 @@ @interface FBMjpegServer() @property (nonatomic, readonly) dispatch_queue_t backgroundQueue; @property (nonatomic, readonly) NSMutableArray *listeningClients; -@property (nonatomic, readonly) FBImageIOScaler *imageScaler; +@property (nonatomic, readonly) FBImageProcessor *imageProcessor; @property (nonatomic, readonly) long long mainScreenID; @end @@ -48,7 +49,7 @@ - (instancetype)init dispatch_async(_backgroundQueue, ^{ [self streamScreenshot]; }); - _imageScaler = [[FBImageIOScaler alloc] init]; + _imageProcessor = [[FBImageProcessor alloc] init]; _mainScreenID = [XCUIScreen.mainScreen displayID]; } return self; @@ -82,15 +83,11 @@ - (void)streamScreenshot } } - CGFloat scalingFactor = [FBConfiguration mjpegScalingFactor] / 100.0f; - BOOL usesScaling = fabs(FBMaxScalingFactor - scalingFactor) > DBL_EPSILON; - CGFloat compressionQuality = FBConfiguration.mjpegServerScreenshotQuality / 100.0f; - // If scaling is applied we perform another JPEG compression after scaling - // To get the desired compressionQuality we need to do a lossless compression here - CGFloat screenshotCompressionQuality = usesScaling ? FBMaxCompressionQuality : compressionQuality; NSError *error; + CGFloat compressionQuality = MAX(FBMinCompressionQuality, + MIN(FBMaxCompressionQuality, FBConfiguration.mjpegServerScreenshotQuality / 100.0)); NSData *screenshotData = [FBScreenshot takeInOriginalResolutionWithScreenID:self.mainScreenID - compressionQuality:screenshotCompressionQuality + compressionQuality:compressionQuality uti:UTTypeJPEG timeout:FRAME_TIMEOUT error:&error]; @@ -100,16 +97,12 @@ - (void)streamScreenshot return; } - if (usesScaling) { - [self.imageScaler submitImage:screenshotData - scalingFactor:scalingFactor - compressionQuality:compressionQuality - completionHandler:^(NSData * _Nonnull scaled) { - [self sendScreenshot:scaled]; - }]; - } else { - [self sendScreenshot:screenshotData]; - } + CGFloat scalingFactor = FBConfiguration.mjpegScalingFactor / 100.0; + [self.imageProcessor submitImageData:screenshotData + scalingFactor:scalingFactor + completionHandler:^(NSData * _Nonnull scaled) { + [self sendScreenshot:scaled]; + }]; [self scheduleNextScreenshotWithInterval:timerInterval timeStarted:timeStarted]; } diff --git a/WebDriverAgentLib/Utilities/FBScreenshot.h b/WebDriverAgentLib/Utilities/FBScreenshot.h index 036a5e1cb..74e6b5886 100644 --- a/WebDriverAgentLib/Utilities/FBScreenshot.h +++ b/WebDriverAgentLib/Utilities/FBScreenshot.h @@ -24,19 +24,6 @@ NS_ASSUME_NONNULL_BEGIN + (nullable NSData *)takeInOriginalResolutionWithQuality:(NSUInteger)quality error:(NSError **)error; -/** - Retrieves non-scaled screenshot of the particular screen rectangle - - @param quality The number in range 0-3, where 0 is PNG (lossless), 3 is HEIC (lossless), 1- low quality JPEG and 2 - high quality JPEG - @param rect The bounding rectange for the screenshot. The value is expected be non-scaled one. - CGRectNull could be used to take a screenshot of the full screen. - @param error If there is an error, upon return contains an NSError object that describes the problem. - @return Device screenshot as PNG-encoded data or nil in case of failure - */ -+ (nullable NSData *)takeInOriginalResolutionWithQuality:(NSUInteger)quality - rect:(CGRect)rect - error:(NSError **)error; - /** Retrieves non-scaled screenshot of the whole screen diff --git a/WebDriverAgentLib/Utilities/FBScreenshot.m b/WebDriverAgentLib/Utilities/FBScreenshot.m index 8cea6530d..3d9716dea 100644 --- a/WebDriverAgentLib/Utilities/FBScreenshot.m +++ b/WebDriverAgentLib/Utilities/FBScreenshot.m @@ -13,7 +13,7 @@ #import "FBConfiguration.h" #import "FBErrorBuilder.h" -#import "FBImageIOScaler.h" +#import "FBImageProcessor.h" #import "FBLogger.h" #import "FBMacros.h" #import "FBXCodeCompatibility.h" @@ -59,30 +59,19 @@ + (UTType *)imageUtiWithQuality:(NSUInteger)quality } + (NSData *)takeInOriginalResolutionWithQuality:(NSUInteger)quality - rect:(CGRect)rect error:(NSError **)error { XCUIScreen *mainScreen = XCUIScreen.mainScreen; return [self.class takeWithScreenID:mainScreen.displayID scale:SCREENSHOT_SCALE compressionQuality:[self.class compressionQualityWithQuality:quality] - rect:rect sourceUTI:[self.class imageUtiWithQuality:quality] error:error]; } -+ (NSData *)takeInOriginalResolutionWithQuality:(NSUInteger)quality - error:(NSError **)error -{ - return [self.class takeInOriginalResolutionWithQuality:quality - rect:CGRectNull - error:error]; -} - + (NSData *)takeWithScreenID:(long long)screenID scale:(CGFloat)scale compressionQuality:(CGFloat)compressionQuality - rect:(CGRect)rect sourceUTI:(UTType *)uti error:(NSError **)error { @@ -94,9 +83,8 @@ + (NSData *)takeWithScreenID:(long long)screenID if (nil == screenshotData) { return nil; } - return [[[FBImageIOScaler alloc] init] scaledImageWithImage:screenshotData + return [[[FBImageProcessor alloc] init] scaledImageWithData:screenshotData uti:UTTypePNG - rect:rect scalingFactor:1.0 / scale compressionQuality:FBMaxCompressionQuality error:error]; diff --git a/WebDriverAgentLib/Utilities/FBSettings.h b/WebDriverAgentLib/Utilities/FBSettings.h index 8235cb337..dd82d740c 100644 --- a/WebDriverAgentLib/Utilities/FBSettings.h +++ b/WebDriverAgentLib/Utilities/FBSettings.h @@ -17,6 +17,7 @@ extern NSString* const FB_SETTING_USE_COMPACT_RESPONSES; extern NSString* const FB_SETTING_ELEMENT_RESPONSE_ATTRIBUTES; extern NSString* const FB_SETTING_MJPEG_SERVER_SCREENSHOT_QUALITY; extern NSString* const FB_SETTING_MJPEG_SERVER_FRAMERATE; +extern NSString* const FB_SETTING_MJPEG_FIX_ORIENTATION; extern NSString* const FB_SETTING_MJPEG_SCALING_FACTOR; extern NSString* const FB_SETTING_SCREENSHOT_QUALITY; extern NSString* const FB_SETTING_KEYBOARD_AUTOCORRECTION; diff --git a/WebDriverAgentLib/Utilities/FBSettings.m b/WebDriverAgentLib/Utilities/FBSettings.m index 26bae07e0..6b12a4cd4 100644 --- a/WebDriverAgentLib/Utilities/FBSettings.m +++ b/WebDriverAgentLib/Utilities/FBSettings.m @@ -14,6 +14,7 @@ NSString* const FB_SETTING_MJPEG_SERVER_SCREENSHOT_QUALITY = @"mjpegServerScreenshotQuality"; NSString* const FB_SETTING_MJPEG_SERVER_FRAMERATE = @"mjpegServerFramerate"; NSString* const FB_SETTING_MJPEG_SCALING_FACTOR = @"mjpegScalingFactor"; +NSString* const FB_SETTING_MJPEG_FIX_ORIENTATION = @"mjpegFixOrientation"; NSString* const FB_SETTING_SCREENSHOT_QUALITY = @"screenshotQuality"; NSString* const FB_SETTING_KEYBOARD_AUTOCORRECTION = @"keyboardAutocorrection"; NSString* const FB_SETTING_KEYBOARD_PREDICTION = @"keyboardPrediction"; diff --git a/WebDriverAgentTests/IntegrationTests/FBImageIOScalerTests.m b/WebDriverAgentTests/IntegrationTests/FBImageProcessorTests.m similarity index 59% rename from WebDriverAgentTests/IntegrationTests/FBImageIOScalerTests.m rename to WebDriverAgentTests/IntegrationTests/FBImageProcessorTests.m index 0c2f75b62..28fa9d7ee 100644 --- a/WebDriverAgentTests/IntegrationTests/FBImageIOScalerTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBImageProcessorTests.m @@ -9,62 +9,62 @@ #import -#import "FBImageIOScaler.h" +#import "FBImageProcessor.h" #import "FBIntegrationTestCase.h" -@interface FBImageIOScalerTests : FBIntegrationTestCase +@interface FBImageProcessorTests : FBIntegrationTestCase @property (nonatomic) NSData *originalImage; @property (nonatomic) CGSize originalSize; @end -@implementation FBImageIOScalerTests +@implementation FBImageProcessorTests - (void)setUp { XCUIApplication *app = [[XCUIApplication alloc] init]; [app launch]; XCUIScreenshot *screenshot = app.screenshot; self.originalImage = UIImageJPEGRepresentation(screenshot.image, 1.0); - self.originalSize = [FBImageIOScalerTests scaledSizeFromImage:screenshot.image]; + self.originalSize = [FBImageProcessorTests scaledSizeFromImage:screenshot.image]; } - (void)testScaling { CGFloat halfScale = 0.5; - CGSize expectedHalfScaleSize = [FBImageIOScalerTests sizeFromSize:self.originalSize scalingFactor:0.5]; + CGSize expectedHalfScaleSize = [FBImageProcessorTests sizeFromSize:self.originalSize scalingFactor:0.5]; [self scaleImageWithFactor:halfScale expectedSize:expectedHalfScaleSize]; - // 1 is the smalles scaling factor we accept + // 0 is the smalles scaling factor we accept CGFloat minScale = 0.0; - CGSize expectedMinScaleSize = [FBImageIOScalerTests sizeFromSize:self.originalSize scalingFactor:0.01]; + CGSize expectedMinScaleSize = [FBImageProcessorTests sizeFromSize:self.originalSize scalingFactor:0.01]; [self scaleImageWithFactor:minScale expectedSize:expectedMinScaleSize]; // For scaling factors above 100 we don't perform any scaling and just return the unmodified image - CGFloat unscaled = 2.0; - [self scaleImageWithFactor:unscaled + [self scaleImageWithFactor:1.0 + expectedSize:self.originalSize]; + [self scaleImageWithFactor:2.0 expectedSize:self.originalSize]; } - (void)scaleImageWithFactor:(CGFloat)scalingFactor expectedSize:(CGSize)excpectedSize { - FBImageIOScaler *scaler = [[FBImageIOScaler alloc] init]; + FBImageProcessor *scaler = [[FBImageProcessor alloc] init]; id expScaled = [self expectationWithDescription:@"Receive scaled image"]; - [scaler submitImage:self.originalImage - scalingFactor:scalingFactor - compressionQuality:1.0 - completionHandler:^(NSData *scaled) { - UIImage *scaledImage = [UIImage imageWithData:scaled]; - CGSize scaledSize = [FBImageIOScalerTests scaledSizeFromImage:scaledImage]; - - XCTAssertEqualWithAccuracy(scaledSize.width, excpectedSize.width, 1.0); - XCTAssertEqualWithAccuracy(scaledSize.height, excpectedSize.height, 1.0); - - [expScaled fulfill]; - }]; + [scaler submitImageData:self.originalImage + scalingFactor:scalingFactor + completionHandler:^(NSData *scaled) { + UIImage *scaledImage = [UIImage imageWithData:scaled]; + CGSize scaledSize = [FBImageProcessorTests scaledSizeFromImage:scaledImage]; + + XCTAssertEqualWithAccuracy(scaledSize.width, excpectedSize.width, 1.0); + XCTAssertEqualWithAccuracy(scaledSize.height, excpectedSize.height, 1.0); + + [expScaled fulfill]; + }]; [self waitForExpectations:@[expScaled] timeout:0.5]; diff --git a/WebDriverAgentTests/IntegrationTests/XCUIDeviceHelperTests.m b/WebDriverAgentTests/IntegrationTests/XCUIDeviceHelperTests.m index 52a910311..ca79b2fa4 100644 --- a/WebDriverAgentTests/IntegrationTests/XCUIDeviceHelperTests.m +++ b/WebDriverAgentTests/IntegrationTests/XCUIDeviceHelperTests.m @@ -78,14 +78,12 @@ - (void)testLandscapeScreenshot XCTAssertTrue(screenshot.size.width > screenshot.size.height); XCUIScreen *mainScreen = XCUIScreen.mainScreen; - // TODO: This screenshot rotation was not landscape in an iOS 16 beta simulator. UIImage *screenshotExact = ((XCUIScreenshot *)mainScreen.screenshot).image; - XCTAssertEqualWithAccuracy(screenshotExact.size.height * mainScreen.scale, - screenshot.size.height, - FLT_EPSILON); - XCTAssertEqualWithAccuracy(screenshotExact.size.width * mainScreen.scale, - screenshot.size.width, - FLT_EPSILON); + CGSize realMainScreenSize = screenshotExact.size.height > screenshot.size.width + ? CGSizeMake(screenshotExact.size.height * mainScreen.scale, screenshotExact.size.width * mainScreen.scale) + : CGSizeMake(screenshotExact.size.width * mainScreen.scale, screenshotExact.size.height * mainScreen.scale); + XCTAssertEqualWithAccuracy(realMainScreenSize.height, screenshot.size.height, FLT_EPSILON); + XCTAssertEqualWithAccuracy(realMainScreenSize.width, screenshot.size.width, FLT_EPSILON); } - (void)testWifiAddress