From c83f7960bcd966cf754ad0e4e99bb45b3d6c3b62 Mon Sep 17 00:00:00 2001 From: dmissmann <37073203+dmissmann@users.noreply.github.com> Date: Tue, 5 Feb 2019 10:14:53 -0800 Subject: [PATCH] Allow image scaling on the mjpeg stream (#138) --- WebDriverAgent.xcodeproj/project.pbxproj | 16 +++ .../Commands/FBSessionCommands.m | 6 + WebDriverAgentLib/Routing/FBWebServer.m | 14 ++ WebDriverAgentLib/Utilities/FBConfiguration.h | 6 + WebDriverAgentLib/Utilities/FBConfiguration.m | 10 ++ WebDriverAgentLib/Utilities/FBImageIOScaler.h | 40 ++++++ WebDriverAgentLib/Utilities/FBImageIOScaler.m | 127 ++++++++++++++++++ WebDriverAgentLib/Utilities/FBMjpegServer.m | 32 ++++- .../IntegrationTests/FBImageIOScalerTests.m | 81 +++++++++++ 9 files changed, 331 insertions(+), 1 deletion(-) create mode 100644 WebDriverAgentLib/Utilities/FBImageIOScaler.h create mode 100644 WebDriverAgentLib/Utilities/FBImageIOScaler.m create mode 100644 WebDriverAgentTests/IntegrationTests/FBImageIOScalerTests.m diff --git a/WebDriverAgent.xcodeproj/project.pbxproj b/WebDriverAgent.xcodeproj/project.pbxproj index 6ee631f70..88740c24b 100644 --- a/WebDriverAgent.xcodeproj/project.pbxproj +++ b/WebDriverAgent.xcodeproj/project.pbxproj @@ -8,6 +8,11 @@ /* Begin PBXBuildFile section */ 1FC3B2E32121ECF600B61EE0 /* FBApplicationProcessProxyTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 1FC3B2E12121EC8C00B61EE0 /* FBApplicationProcessProxyTests.m */; }; + 63CCF91221ECE4C700E94ABD /* FBImageIOScaler.h in Headers */ = {isa = PBXBuildFile; fileRef = 63CCF91021ECE4C700E94ABD /* FBImageIOScaler.h */; }; + 63CCF91321ECE4C700E94ABD /* FBImageIOScaler.m in Sources */ = {isa = PBXBuildFile; fileRef = 63CCF91121ECE4C700E94ABD /* FBImageIOScaler.m */; }; + 63FD950221F9D06100A3E356 /* FBImageIOScalerTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 631B523421F6174300625362 /* FBImageIOScalerTests.m */; }; + 63FD950321F9D06100A3E356 /* FBImageIOScalerTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 631B523421F6174300625362 /* FBImageIOScalerTests.m */; }; + 63FD950421F9D06200A3E356 /* FBImageIOScalerTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 631B523421F6174300625362 /* FBImageIOScalerTests.m */; }; 710181F8211DF584002FD3A8 /* CocoaAsyncSocket.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 710181F7211DF584002FD3A8 /* CocoaAsyncSocket.framework */; }; 71018201211DF62C002FD3A8 /* libxml2.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = 71018200211DF62C002FD3A8 /* libxml2.tbd */; }; 7101820D211E026B002FD3A8 /* libAccessibility.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = 7101820C211E026B002FD3A8 /* libAccessibility.tbd */; }; @@ -453,6 +458,9 @@ 1BA7DD8C206D694B007C7C26 /* XCTElementSetTransformer-Protocol.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "XCTElementSetTransformer-Protocol.h"; sourceTree = ""; }; 1FC3B2E12121EC8C00B61EE0 /* FBApplicationProcessProxyTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBApplicationProcessProxyTests.m; sourceTree = ""; }; 44757A831D42CE8300ECF35E /* XCUIDeviceRotationTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = XCUIDeviceRotationTests.m; sourceTree = ""; }; + 631B523421F6174300625362 /* FBImageIOScalerTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FBImageIOScalerTests.m; sourceTree = ""; }; + 63CCF91021ECE4C700E94ABD /* FBImageIOScaler.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FBImageIOScaler.h; sourceTree = ""; }; + 63CCF91121ECE4C700E94ABD /* FBImageIOScaler.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FBImageIOScaler.m; sourceTree = ""; }; 710181F7211DF584002FD3A8 /* CocoaAsyncSocket.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CocoaAsyncSocket.framework; path = Carthage/Build/iOS/CocoaAsyncSocket.framework; sourceTree = ""; }; 71018200211DF62C002FD3A8 /* libxml2.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libxml2.tbd; path = usr/lib/libxml2.tbd; sourceTree = SDKROOT; }; 7101820C211E026B002FD3A8 /* libAccessibility.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libAccessibility.tbd; path = usr/lib/libAccessibility.tbd; sourceTree = SDKROOT; }; @@ -1154,6 +1162,8 @@ 711084431DA3AA7500F913D6 /* FBXPath.m */, EE6B64FB1D0F86EF00E85F5D /* XCTestPrivateSymbols.h */, EE6B64FC1D0F86EF00E85F5D /* XCTestPrivateSymbols.m */, + 63CCF91021ECE4C700E94ABD /* FBImageIOScaler.h */, + 63CCF91121ECE4C700E94ABD /* FBImageIOScaler.m */, ); name = Utilities; path = WebDriverAgentLib/Utilities; @@ -1213,6 +1223,7 @@ 71E504941DF59BAD0020C32A /* XCUIElementAttributesTests.m */, EEBBD48D1D4785FC00656A81 /* XCUIElementFBFindTests.m */, EE1E06E11D181CC9007CF043 /* XCUIElementHelperIntegrationTests.m */, + 631B523421F6174300625362 /* FBImageIOScalerTests.m */, ); path = IntegrationTests; sourceTree = ""; @@ -1560,6 +1571,7 @@ 71BD20731F86116100B36EC2 /* XCUIApplication+FBTouchAction.h in Headers */, EE158ACE1CBD456F00A3E3F0 /* FBCommandHandler.h in Headers */, EE158AC81CBD456F00A3E3F0 /* FBSessionCommands.h in Headers */, + 63CCF91221ECE4C700E94ABD /* FBImageIOScaler.h in Headers */, EE158AE31CBD456F00A3E3F0 /* FBSession-Private.h in Headers */, 716E0BCE1E917E810087A825 /* NSString+FBXMLSafeString.h in Headers */, EE158ACF1CBD456F00A3E3F0 /* FBCommandStatus.h in Headers */, @@ -1921,6 +1933,7 @@ EEE3764A1D59FAE900ED88DD /* XCUIElement+FBWebDriverAttributes.m in Sources */, EE8DDD7E20C5733C004D4925 /* XCUIElement+FBForceTouch.m in Sources */, 71241D7C1FAE3D2500B9559F /* FBTouchActionCommands.m in Sources */, + 63CCF91321ECE4C700E94ABD /* FBImageIOScaler.m in Sources */, EE158ACB1CBD456F00A3E3F0 /* FBTouchIDCommands.m in Sources */, EE158ABD1CBD456F00A3E3F0 /* FBDebugCommands.m in Sources */, 716E0BCF1E917E810087A825 /* NSString+FBXMLSafeString.m in Sources */, @@ -1967,6 +1980,7 @@ files = ( 71241D801FAF087500B9559F /* FBW3CMultiTouchActionsIntegrationTests.m in Sources */, 71241D7E1FAF084E00B9559F /* FBW3CTouchActionsIntegrationTests.m in Sources */, + 63FD950221F9D06100A3E356 /* FBImageIOScalerTests.m in Sources */, 719CD8FF2126C90200C7D0C2 /* FBAutoAlertsHandlerTests.m in Sources */, EE2202131ECC612200A29571 /* FBIntegrationTestCase.m in Sources */, 71BD20781F869E0F00B36EC2 /* FBAppiumTouchActionsIntegrationTests.m in Sources */, @@ -1983,6 +1997,7 @@ buildActionMask = 2147483647; files = ( EE5095E51EBCC9090028E2FE /* FBTypingTest.m in Sources */, + 63FD950321F9D06100A3E356 /* FBImageIOScalerTests.m in Sources */, EE5095EB1EBCC9090028E2FE /* XCElementSnapshotHitPointTests.m in Sources */, EE5095EC1EBCC9090028E2FE /* XCUIApplicationHelperTests.m in Sources */, EE5095ED1EBCC9090028E2FE /* XCElementSnapshotHelperTests.m in Sources */, @@ -2046,6 +2061,7 @@ EE26409D1D0EBA25009BE6B0 /* FBElementAttributeTests.m in Sources */, 7119E1EC1E891F8600D0B125 /* FBPickerWheelSelectTests.m in Sources */, EE1E06DA1D1808C2007CF043 /* FBIntegrationTestCase.m in Sources */, + 63FD950421F9D06200A3E356 /* FBImageIOScalerTests.m in Sources */, EE05BAFA1D13003C00A3EB00 /* FBKeyboardTests.m in Sources */, EE55B3271D1D54CF003AAAEC /* FBScrollingTests.m in Sources */, EE6A89371D0B35920083E92B /* FBFailureProofTestCaseTests.m in Sources */, diff --git a/WebDriverAgentLib/Commands/FBSessionCommands.m b/WebDriverAgentLib/Commands/FBSessionCommands.m index e740d26d1..09521e347 100644 --- a/WebDriverAgentLib/Commands/FBSessionCommands.m +++ b/WebDriverAgentLib/Commands/FBSessionCommands.m @@ -23,6 +23,8 @@ static NSString* const ELEMENT_RESPONSE_ATTRIBUTES = @"elementResponseAttributes"; static NSString* const MJPEG_SERVER_SCREENSHOT_QUALITY = @"mjpegServerScreenshotQuality"; static NSString* const MJPEG_SERVER_FRAMERATE = @"mjpegServerFramerate"; +static NSString* const MJPEG_SCALING_FACTOR = @"mjpegScalingFactor"; +static NSString* const MJPEG_COMPRESSION_FACTOR = @"mjpegCompressionFactor"; static NSString* const SCREENSHOT_QUALITY = @"screenshotQuality"; @implementation FBSessionCommands @@ -204,6 +206,7 @@ + (NSArray *)routes ELEMENT_RESPONSE_ATTRIBUTES: [FBConfiguration elementResponseAttributes], MJPEG_SERVER_SCREENSHOT_QUALITY: @([FBConfiguration mjpegServerScreenshotQuality]), MJPEG_SERVER_FRAMERATE: @([FBConfiguration mjpegServerFramerate]), + MJPEG_SCALING_FACTOR: @([FBConfiguration mjpegScalingFactor]), SCREENSHOT_QUALITY: @([FBConfiguration screenshotQuality]), } ); @@ -230,6 +233,9 @@ + (NSArray *)routes if ([settings objectForKey:SCREENSHOT_QUALITY]) { [FBConfiguration setScreenshotQuality:[[settings objectForKey:SCREENSHOT_QUALITY] unsignedIntegerValue]]; } + if ([settings objectForKey:MJPEG_SCALING_FACTOR]) { + [FBConfiguration setMjpegScalingFactor:[[settings objectForKey:MJPEG_SCALING_FACTOR] unsignedIntegerValue]]; + } return [self handleGetSettings:request]; } diff --git a/WebDriverAgentLib/Routing/FBWebServer.m b/WebDriverAgentLib/Routing/FBWebServer.m index f3a180b34..80b4205b7 100644 --- a/WebDriverAgentLib/Routing/FBWebServer.m +++ b/WebDriverAgentLib/Routing/FBWebServer.m @@ -115,6 +115,7 @@ - (void)startHTTPServer - (void)initScreenshotsBroadcaster { + [self readMjpegSettingsFromEnv]; self.screenshotsBroadcaster = [[FBTCPSocket alloc] initWithPort:(uint16_t)FBConfiguration.mjpegServerPort]; self.screenshotsBroadcaster.delegate = [[FBMjpegServer alloc] init]; @@ -134,6 +135,19 @@ - (void)stopScreenshotsBroadcaster [self.screenshotsBroadcaster stop]; } +- (void)readMjpegSettingsFromEnv +{ + NSDictionary *env = NSProcessInfo.processInfo.environment; + NSString *scalingFactor = [env objectForKey:@"MJPEG_SCALING_FACTOR"]; + if (scalingFactor != nil && [scalingFactor length] > 0) { + [FBConfiguration setMjpegScalingFactor:[scalingFactor integerValue]]; + } + NSString *screenshotQuality = [env objectForKey:@"MJPEG_SERVER_SCREENSHOT_QUALITY"]; + if (screenshotQuality != nil && [screenshotQuality length] > 0) { + [FBConfiguration setMjpegServerScreenshotQuality:[screenshotQuality integerValue]]; + } +} + - (void)stopServing { [FBSession.activeSession kill]; diff --git a/WebDriverAgentLib/Utilities/FBConfiguration.h b/WebDriverAgentLib/Utilities/FBConfiguration.h index 8ab5ff4ec..43ef01a67 100644 --- a/WebDriverAgentLib/Utilities/FBConfiguration.h +++ b/WebDriverAgentLib/Utilities/FBConfiguration.h @@ -92,6 +92,12 @@ NS_ASSUME_NONNULL_BEGIN */ + (NSInteger)mjpegServerPort; +/** + The scaling factor for frames of the mjpeg stream (Default values is 100 and does not perform scaling). + */ ++ (NSUInteger)mjpegScalingFactor; ++ (void)setMjpegScalingFactor:(NSUInteger)scalingFactor; + /** YES if verbose logging is enabled. NO otherwise. */ diff --git a/WebDriverAgentLib/Utilities/FBConfiguration.m b/WebDriverAgentLib/Utilities/FBConfiguration.m index cf73e8733..f3fe5e6b2 100644 --- a/WebDriverAgentLib/Utilities/FBConfiguration.m +++ b/WebDriverAgentLib/Utilities/FBConfiguration.m @@ -29,6 +29,7 @@ static NSUInteger FBMjpegServerScreenshotQuality = 25; static NSUInteger FBMjpegServerFramerate = 10; static NSUInteger FBScreenshotQuality = 1; +static NSUInteger FBMjpegScalingFactor = 100; @implementation FBConfiguration @@ -74,6 +75,15 @@ + (NSInteger)mjpegServerPort return DefaultMjpegServerPort; } ++ (NSUInteger)mjpegScalingFactor +{ + return FBMjpegScalingFactor; +} + ++ (void)setMjpegScalingFactor:(NSUInteger)scalingFactor { + FBMjpegScalingFactor = scalingFactor; +} + + (BOOL)verboseLoggingEnabled { return [NSProcessInfo.processInfo.environment[@"VERBOSE_LOGGING"] boolValue]; diff --git a/WebDriverAgentLib/Utilities/FBImageIOScaler.h b/WebDriverAgentLib/Utilities/FBImageIOScaler.h new file mode 100644 index 000000000..b5e6d42d5 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBImageIOScaler.h @@ -0,0 +1,40 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +// Those values define the allowed ranges for the scaling factor and compression quality settings +extern const CGFloat FBMinScalingFactor; +extern const CGFloat FBMaxScalingFactor; +extern const CGFloat FBMinCompressionQuality; +extern const CGFloat FBMaxCompressionQuality; + + +/** + Scales images and compresses it to JPEG using Image I/O + It allows to enqueue only a single screenshot. If a new one arrives before the currently queued gets discared + */ +@interface FBImageIOScaler : NSObject + +/** + Puts the passed image on the queue and dispatches a scaling operation. If there is already a image on the + queue it will be replaced with the new one + @param image The image to scale down + @param completionHandler called after successfully scaling down an image + @param scalingFactor the scaling factor in range 0.01..1.0. A value of 1.0 won't perform scaling at all + @param compressionQuality the compression quality in range 0.0..1.0 (0.0 for max. compression and 1.0 for lossless compression) + */ +- (void)submitImage:(NSData *)image scalingFactor:(CGFloat)scalingFactor compressionQuality:(CGFloat)compressionQuality completionHandler:(void (^)(NSData *))completionHandler; + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBImageIOScaler.m b/WebDriverAgentLib/Utilities/FBImageIOScaler.m new file mode 100644 index 000000000..e7857bcf1 --- /dev/null +++ b/WebDriverAgentLib/Utilities/FBImageIOScaler.m @@ -0,0 +1,127 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#import "FBImageIOScaler.h" +#import +#import +#import "FBLogger.h" + +const CGFloat FBMinScalingFactor = 0.01f; +const CGFloat FBMaxScalingFactor = 1.0f; +const CGFloat FBMinCompressionQuality = 0.0f; +const CGFloat FBMaxCompressionQuality = 1.0f; + +@interface FBImageIOScaler () + +@property (nonatomic) NSData *nextImage; +@property (nonatomic, readonly) NSLock *nextImageLock; +@property (nonatomic, readonly) dispatch_queue_t scalingQueue; + +@end + +@implementation FBImageIOScaler + +- (id)init +{ + self = [super init]; + if (self) { + _nextImageLock = [[NSLock alloc] init]; + _scalingQueue = dispatch_queue_create("image.scaling.queue", NULL); + } + return self; +} + +- (void)submitImage:(NSData *)image scalingFactor:(CGFloat)scalingFactor compressionQuality:(CGFloat)compressionQuality completionHandler:(void (^)(NSData *))completionHandler { + [self.nextImageLock lock]; + if (self.nextImage != nil) { + [FBLogger verboseLog:@"Discarding screenshot"]; + } + scalingFactor = MAX(FBMinScalingFactor, MIN(FBMaxScalingFactor, scalingFactor)); + compressionQuality = MAX(FBMinCompressionQuality, MIN(FBMaxCompressionQuality, compressionQuality)); + self.nextImage = image; + [self.nextImageLock unlock]; + + dispatch_async(self.scalingQueue, ^{ + [self.nextImageLock lock]; + NSData *next = self.nextImage; + self.nextImage = nil; + [self.nextImageLock unlock]; + if (next == nil) { + return; + } + NSData *scaled = [self scaledImageWithImage:next + scalingFactor:scalingFactor + compressionQuality:compressionQuality]; + if (scaled == nil) { + [FBLogger log:@"Could not scale down the image"]; + return; + } + completionHandler(scaled); + }); +} + +- (nullable NSData *)scaledImageWithImage:(NSData *)image scalingFactor:(CGFloat)scalingFactor compressionQuality:(CGFloat)compressionQuality { + CGImageSourceRef imageData = CGImageSourceCreateWithData((CFDataRef)image, nil); + + CGSize size = [FBImageIOScaler imageSizeWithImage:imageData]; + CGFloat scaledMaxPixelSize = MAX(size.width, size.height) * scalingFactor; + + CFDictionaryRef params = (__bridge CFDictionaryRef)@{ + (const NSString *)kCGImageSourceCreateThumbnailWithTransform: @(YES), + (const NSString *)kCGImageSourceCreateThumbnailFromImageIfAbsent: @(YES), + (const NSString *)kCGImageSourceThumbnailMaxPixelSize: @(scaledMaxPixelSize) + }; + + CGImageRef scaled = CGImageSourceCreateThumbnailAtIndex(imageData, 0, params); + if (scaled == nil) { + [FBLogger log:@"Failed to scale the image"]; + CFRelease(imageData); + return nil; + } + NSData *jpegData = [self jpegDataWithImage:scaled + compressionQuality:compressionQuality]; + CGImageRelease(scaled); + CFRelease(imageData); + return jpegData; +} + +- (nullable NSData *)jpegDataWithImage:(CGImageRef)imageRef compressionQuality:(CGFloat)compressionQuality +{ + NSMutableData *newImageData = [NSMutableData data]; + CGImageDestinationRef imageDestination = CGImageDestinationCreateWithData((CFMutableDataRef)newImageData, kUTTypeJPEG, 1, NULL); + + CFDictionaryRef compressionOptions = (__bridge CFDictionaryRef)@{ + (const NSString *)kCGImageDestinationLossyCompressionQuality: @(compressionQuality) + }; + + CGImageDestinationAddImage(imageDestination, imageRef, compressionOptions); + if(!CGImageDestinationFinalize(imageDestination)) { + [FBLogger log:@"Failed to write the image"]; + newImageData = nil; + } + CFRelease(imageDestination); + return newImageData; +} + ++ (CGSize)imageSizeWithImage:(CGImageSourceRef)imageSource +{ + NSDictionary *options = @{ + (const NSString *)kCGImageSourceShouldCache: @(NO) + }; + CFDictionaryRef properties = CGImageSourceCopyPropertiesAtIndex(imageSource, 0, (CFDictionaryRef)options); + + NSNumber *width = [(__bridge NSDictionary *)properties objectForKey:(const NSString *)kCGImagePropertyPixelWidth]; + NSNumber *height = [(__bridge NSDictionary *)properties objectForKey:(const NSString *)kCGImagePropertyPixelHeight]; + + CGSize size = CGSizeMake([width floatValue], [height floatValue]); + CFRelease(properties); + return size; +} + +@end diff --git a/WebDriverAgentLib/Utilities/FBMjpegServer.m b/WebDriverAgentLib/Utilities/FBMjpegServer.m index b73e32edb..4a03a3bef 100644 --- a/WebDriverAgentLib/Utilities/FBMjpegServer.m +++ b/WebDriverAgentLib/Utilities/FBMjpegServer.m @@ -19,6 +19,7 @@ #import "XCTestManager_ManagerInterface-Protocol.h" #import "FBXCTestDaemonsProxy.h" #import "XCUIScreen.h" +#import "FBImageIOScaler.h" static const NSTimeInterval SCREENSHOT_TIMEOUT = 0.5; static const NSUInteger MAX_FPS = 60; @@ -32,6 +33,7 @@ @interface FBMjpegServer() @property (nonatomic, readonly) dispatch_queue_t backgroundQueue; @property (nonatomic, readonly) NSMutableArray *activeClients; @property (nonatomic, readonly) mach_timebase_info_data_t timebaseInfo; +@property (nonatomic, readonly) FBImageIOScaler *imageScaler; @end @@ -48,6 +50,7 @@ - (instancetype)init dispatch_async(_backgroundQueue, ^{ [self streamScreenshot]; }); + _imageScaler = [[FBImageIOScaler alloc] init]; } return self; } @@ -86,7 +89,17 @@ - (void)streamScreenshot } __block NSData *screenshotData = nil; + + CGFloat scalingFactor = [FBConfiguration mjpegScalingFactor] / 100.0f; + BOOL usesScaling = fabs(FBMaxScalingFactor - scalingFactor) > DBL_EPSILON; + CGFloat compressionQuality = FBConfiguration.mjpegServerScreenshotQuality / 100.0f; + + // If scaling is applied we perform another JPEG compression after scaling + // To get the desired compressionQuality we need to do a lossless compression here + if (usesScaling) { + compressionQuality = FBMaxScalingFactor; + } id proxy = [FBXCTestDaemonsProxy testRunnerProxy]; dispatch_semaphore_t sem = dispatch_semaphore_create(0); [proxy _XCT_requestScreenshotOfScreenWithID:[[XCUIScreen mainScreen] displayID] @@ -94,6 +107,9 @@ - (void)streamScreenshot uti:(__bridge id)kUTTypeJPEG compressionQuality:compressionQuality withReply:^(NSData *data, NSError *error) { + if (error != nil) { + [FBLogger logFmt:@"Error taking screenshot: %@", [error description]]; + } screenshotData = data; dispatch_semaphore_signal(sem); }]; @@ -103,6 +119,21 @@ - (void)streamScreenshot return; } + if (usesScaling) { + [self.imageScaler submitImage:screenshotData + scalingFactor:scalingFactor + compressionQuality:compressionQuality + completionHandler:^(NSData * _Nonnull scaled) { + [self sendScreenshot:scaled]; + }]; + } else { + [self sendScreenshot:screenshotData]; + } + + [self scheduleNextScreenshotWithInterval:timerInterval timeStarted:timeStarted]; +} + +- (void)sendScreenshot:(NSData *)screenshotData { NSString *chunkHeader = [NSString stringWithFormat:@"--BoundaryString\r\nContent-type: image/jpg\r\nContent-Length: %@\r\n\r\n", @(screenshotData.length)]; NSMutableData *chunk = [[chunkHeader dataUsingEncoding:NSUTF8StringEncoding] mutableCopy]; [chunk appendData:screenshotData]; @@ -112,7 +143,6 @@ - (void)streamScreenshot [client writeData:chunk withTimeout:-1 tag:0]; } } - [self scheduleNextScreenshotWithInterval:timerInterval timeStarted:timeStarted]; } + (BOOL)canStreamScreenshots diff --git a/WebDriverAgentTests/IntegrationTests/FBImageIOScalerTests.m b/WebDriverAgentTests/IntegrationTests/FBImageIOScalerTests.m new file mode 100644 index 000000000..6f6636956 --- /dev/null +++ b/WebDriverAgentTests/IntegrationTests/FBImageIOScalerTests.m @@ -0,0 +1,81 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#import +#import "FBImageIOScaler.h" +#import "FBIntegrationTestCase.h" + +@interface FBImageIOScalerTests : FBIntegrationTestCase + +@property (nonatomic) NSData *originalImage; +@property (nonatomic) CGSize originalSize; + +@end + +@implementation FBImageIOScalerTests + +- (void)setUp { + XCUIApplication *app = [[XCUIApplication alloc] init]; + [app launch]; + XCUIScreenshot *screenshot = app.screenshot; + self.originalImage = UIImageJPEGRepresentation(screenshot.image, 1.0); + self.originalSize = [FBImageIOScalerTests scaledSizeFromImage:screenshot.image]; +} + +- (void)testScaling { + CGFloat halfScale = 0.5; + CGSize expectedHalfScaleSize = [FBImageIOScalerTests sizeFromSize:self.originalSize scalingFactor:0.5]; + [self scaleImageWithFactor:halfScale + expectedSize:expectedHalfScaleSize]; + + // 1 is the smalles scaling factor we accept + CGFloat minScale = 0.0; + CGSize expectedMinScaleSize = [FBImageIOScalerTests sizeFromSize:self.originalSize scalingFactor:0.01]; + [self scaleImageWithFactor:minScale + expectedSize:expectedMinScaleSize]; + + // For scaling factors above 100 we don't perform any scaling and just return the unmodified image + CGFloat unscaled = 2.0; + [self scaleImageWithFactor:unscaled + expectedSize:self.originalSize]; +} + +- (void)scaleImageWithFactor:(CGFloat)scalingFactor expectedSize:(CGSize)excpectedSize { + FBImageIOScaler *scaler = [[FBImageIOScaler alloc] init]; + + id expScaled = [self expectationWithDescription:@"Receive scaled image"]; + + [scaler submitImage:self.originalImage + scalingFactor:scalingFactor + compressionQuality:1.0 + completionHandler:^(NSData *scaled) { + UIImage *scaledImage = [UIImage imageWithData:scaled]; + CGSize scaledSize = [FBImageIOScalerTests scaledSizeFromImage:scaledImage]; + + XCTAssertEqualWithAccuracy(scaledSize.width, excpectedSize.width, DBL_EPSILON); + XCTAssertEqualWithAccuracy(scaledSize.height, excpectedSize.height, DBL_EPSILON); + + [expScaled fulfill]; + }]; + + [self waitForExpectations:@[expScaled] + timeout:0.5]; + +} + ++ (CGSize)scaledSizeFromImage:(UIImage *)image { + return CGSizeMake(image.size.width * image.scale, image.size.height * image.scale); +} + ++ (CGSize)sizeFromSize:(CGSize)size scalingFactor:(CGFloat)scalingFactor { + return CGSizeMake(round(size.width * scalingFactor), round(size.height * scalingFactor)); +} + +@end +