Implement SDL 0179 - Pixel density and Scale

SmartDeviceLink/SDLCarWindow.m

@@ -21,6 +21,7 @@
 #import "SDLStreamingMediaConfiguration.h"
 #import "SDLStreamingVideoLifecycleManager.h"
 #import "SDLStreamingMediaManagerConstants.h"
+#import "SDLVideoStreamingCapability.h"
 
 NS_ASSUME_NONNULL_BEGIN
 
@@ -71,7 +72,7 @@ - (void)syncFrame {
         return;
     }
 
-    CGRect bounds = self.rootViewController.view.bounds;
+    CGRect bounds = self.sdl_getScaledScreenSizeFrame;
 
     UIGraphicsBeginImageContextWithOptions(bounds.size, YES, 1.0f);
     switch (self.renderingType) {
@@ -120,13 +121,18 @@ - (void)sdl_didReceiveVideoStreamStarted:(NSNotification *)notification {
 
     dispatch_async(dispatch_get_main_queue(), ^{
         // If the video stream has started, we want to resize the streamingViewController to the size from the RegisterAppInterface
-        self.rootViewController.view.frame = CGRectMake(0, 0, self.streamManager.screenSize.width, self.streamManager.screenSize.height);
+        self.rootViewController.view.frame = self.sdl_getScaledScreenSizeFrame;
         self.rootViewController.view.bounds = self.rootViewController.view.frame;
-
+        
         SDLLogD(@"Video stream started, setting CarWindow frame to: %@", NSStringFromCGRect(self.rootViewController.view.bounds));
     });
 }
 
+- (CGRect)sdl_getScaledScreenSizeFrame {
+    float scale = self.streamManager.videoStreamingCapability.scale.floatValue;
+    return CGRectMake(0, 0, self.streamManager.screenSize.width / scale, self.streamManager.screenSize.height / scale);
+}
+
 - (void)sdl_didReceiveVideoStreamStopped:(NSNotification *)notification {
     self.videoStreamStarted = false;
 

SmartDeviceLink/SDLRPCParameterNames.h

@@ -139,6 +139,7 @@ extern SDLRPCParameterName const SDLRPCParameterNameDestinationETA;
 extern SDLRPCParameterName const SDLRPCParameterNameDetails;
 extern SDLRPCParameterName const SDLRPCParameterNameDeviceInfo;
 extern SDLRPCParameterName const SDLRPCParameterNameDeviceStatus;
+extern SDLRPCParameterName const SDLRPCParameterNameDiagonalScreenSize;
 extern SDLRPCParameterName const SDLRPCParameterNameDialNumberEnabled;
 extern SDLRPCParameterName const SDLRPCParameterNameDIDLocation;
 extern SDLRPCParameterName const SDLRPCParameterNameDIDResult;
@@ -443,6 +444,7 @@ extern SDLRPCParameterName const SDLRPCParameterNamePhoneCall;
 extern SDLRPCParameterName const SDLRPCParameterNamePhoneCapability;
 extern SDLRPCParameterName const SDLRPCParameterNamePhoneNumber;
 extern SDLRPCParameterName const SDLRPCParameterNamePhoneRoaming;
+extern SDLRPCParameterName const SDLRPCParameterNamePixelPerInch;
 extern SDLRPCParameterName const SDLRPCParameterNamePlaylistName;
 extern SDLRPCParameterName const SDLRPCParameterNamePlayTone;
 extern SDLRPCParameterName const SDLRPCParameterNamePosition;
@@ -514,6 +516,7 @@ extern SDLRPCParameterName const SDLRPCParameterNameRPM;
 extern SDLRPCParameterName const SDLRPCParameterNameRadioText;
 extern SDLRPCParameterName const SDLRPCParameterNameSamplingRate;
 extern SDLRPCParameterName const SDLRPCParameterNameSatellites;
+extern SDLRPCParameterName const SDLRPCParameterNameScale;
 extern SDLRPCParameterName const SDLRPCParameterNameScreenParams;
 extern SDLRPCParameterName const SDLRPCParameterNameScrollableMessageBody;
 extern SDLRPCParameterName const SDLRPCParameterNameSDLVersion;

SmartDeviceLink/SDLRPCParameterNames.m

@@ -137,6 +137,7 @@
 SDLRPCParameterName const SDLRPCParameterNameDeviceInfo = @"deviceInfo";
 SDLRPCParameterName const SDLRPCParameterNameDeviceStatus = @"deviceStatus";
 SDLRPCParameterName const SDLRPCParameterNameDetails = @"details";
+SDLRPCParameterName const SDLRPCParameterNameDiagonalScreenSize = @"diagonalScreenSize";
 SDLRPCParameterName const SDLRPCParameterNameDialNumberEnabled = @"dialNumberEnabled";
 SDLRPCParameterName const SDLRPCParameterNameDIDLocation = @"didLocation";
 SDLRPCParameterName const SDLRPCParameterNameDIDResult = @"didResult";
@@ -437,6 +438,7 @@
 SDLRPCParameterName const SDLRPCParameterNamePhoneCapability = @"phoneCapability";
 SDLRPCParameterName const SDLRPCParameterNamePhoneNumber = @"phoneNumber";
 SDLRPCParameterName const SDLRPCParameterNamePhoneRoaming = @"phoneRoaming";
+SDLRPCParameterName const SDLRPCParameterNamePixelPerInch = @"pixelPerInch";
 SDLRPCParameterName const SDLRPCParameterNamePrimaryColor = @"primaryColor";
 SDLRPCParameterName const SDLRPCParameterNamePlaylistName = @"playlistName";
 SDLRPCParameterName const SDLRPCParameterNamePlayTone = @"playTone";
@@ -509,6 +511,7 @@
 SDLRPCParameterName const SDLRPCParameterNameRadioText  = @"RT";
 SDLRPCParameterName const SDLRPCParameterNameSamplingRate = @"samplingRate";
 SDLRPCParameterName const SDLRPCParameterNameSatellites = @"satellites";
+SDLRPCParameterName const SDLRPCParameterNameScale = @"scale";
 SDLRPCParameterName const SDLRPCParameterNameScreenParams = @"screenParams";
 SDLRPCParameterName const SDLRPCParameterNameScrollableMessageBody = @"scrollableMessageBody";
 SDLRPCParameterName const SDLRPCParameterNameSDLVersion = @"sdlVersion";

SmartDeviceLink/SDLStreamingVideoLifecycleManager.h

@@ -22,6 +22,7 @@
 @class SDLStateMachine;
 @class SDLStreamingMediaConfiguration;
 @class SDLTouchManager;
+@class SDLVideoStreamingCapability;
 
 @protocol SDLConnectionManagerType;
 @protocol SDLFocusableItemLocatorType;
@@ -35,6 +36,7 @@ NS_ASSUME_NONNULL_BEGIN
 
 @property (strong, nonatomic, readonly) SDLStateMachine *videoStreamStateMachine;
 @property (strong, nonatomic, readonly) SDLVideoStreamManagerState *currentVideoStreamState;
+@property (nullable, strong, nonatomic, readonly) SDLVideoStreamingCapability *videoStreamingCapability;
 
 @property (strong, nonatomic, readonly) SDLStateMachine *appStateMachine;
 @property (strong, nonatomic, readonly) SDLAppState *currentAppState;

SmartDeviceLink/SDLStreamingVideoLifecycleManager.m

@@ -58,6 +58,8 @@ @interface SDLStreamingVideoLifecycleManager() <SDLVideoEncoderDelegate>
 @property (assign, nonatomic, readonly, getter=isAppStateVideoStreamCapable) BOOL appStateVideoStreamCapable;
 @property (assign, nonatomic, readonly, getter=isHmiStateVideoStreamCapable) BOOL hmiStateVideoStreamCapable;
 
+@property (nullable, strong, nonatomic, readwrite) SDLVideoStreamingCapability *videoStreamingCapability;
+
 @property (strong, nonatomic, readwrite) SDLStateMachine *videoStreamStateMachine;
 @property (strong, nonatomic, readwrite) SDLStateMachine *appStateMachine;
 
@@ -393,7 +395,9 @@ - (void)didEnterStateVideoStreamReady {
         NSAssert(self.videoFormat != nil, @"No video format is known, but it must be if we got a protocol start service response");
 
         SDLLogD(@"Attempting to create video encoder");
-        self.videoEncoder = [[SDLH264VideoEncoder alloc] initWithProtocol:self.videoFormat.protocol dimensions:self.screenSize ssrc:self.ssrc properties:self.videoEncoderSettings delegate:self error:&error];
+        float scale = self.videoStreamingCapability.scale.floatValue;
+        CGSize scaledScreenSize = CGSizeMake(self.screenSize.width / scale, self.screenSize.height / scale);
+        self.videoEncoder = [[SDLH264VideoEncoder alloc] initWithProtocol:self.videoFormat.protocol dimensions:scaledScreenSize ssrc:self.ssrc properties:self.videoEncoderSettings delegate:self error:&error];
 
         if (error || self.videoEncoder == nil) {
             SDLLogE(@"Could not create a video encoder: %@", error);
@@ -710,6 +714,8 @@ - (void)sdl_requestVideoCapabilities:(SDLVideoCapabilityResponseHandler)response
         }
 
         SDLVideoStreamingCapability *videoCapability = ((SDLGetSystemCapabilityResponse *)response).systemCapability.videoStreamingCapability;
+        self.videoStreamingCapability = videoCapability;
+        self.touchManager.videoStreamingCapability = videoCapability;
         SDLLogD(@"Video capabilities response received: %@", videoCapability);
         responseHandler(videoCapability);
     }];

SmartDeviceLink/SDLTouchManager.h

@@ -14,6 +14,7 @@
 @protocol SDLTouchManagerDelegate;
 
 @class SDLTouch;
+@class SDLVideoStreamingCapability;
 
 NS_ASSUME_NONNULL_BEGIN
 
@@ -78,6 +79,11 @@ typedef void(^SDLTouchEventHandler)(SDLTouch *touch, SDLTouchType type);
  */
 @property (nonatomic, assign, getter=isTouchEnabled) BOOL touchEnabled;
 
+/**
+ Provides all video streaming capabilities defined in the HMI.
+ */
+@property (nullable, strong, nonatomic) SDLVideoStreamingCapability *videoStreamingCapability;
+
 /**
  *  @abstract
  *      Cancels pending touch event timers that may be in progress.

SmartDeviceLink/SDLTouchManager.m

@@ -22,6 +22,7 @@
 #import "SDLTouchCoord.h"
 #import "SDLTouchEvent.h"
 #import "SDLTouchManagerDelegate.h"
+#import "SDLVideoStreamingCapability.h"
 
 
 NS_ASSUME_NONNULL_BEGIN
@@ -184,7 +185,8 @@ - (void)sdl_onTouchEvent:(SDLRPCNotificationNotification *)notification {
         return;
     }
 
-    SDLOnTouchEvent* onTouchEvent = (SDLOnTouchEvent*)notification.notification;
+    SDLOnTouchEvent *onTouchEvent = (SDLOnTouchEvent *)notification.notification;
+    onTouchEvent = [self sdl_applyScaleToEventCoordinates:onTouchEvent.copy];
 
     SDLTouchType touchType = onTouchEvent.type;
     [onTouchEvent.event enumerateObjectsUsingBlock:^(SDLTouchEvent *touchEvent, NSUInteger idx, BOOL *stop) {
@@ -212,6 +214,26 @@ - (void)sdl_onTouchEvent:(SDLRPCNotificationNotification *)notification {
     }];
 }
 
+/**
+ *  Modifies the existing coordinates of the SDLOnTouchEvent, based on the received 'scale' value.
+
+ *  This will match the coordinates to the scaled resolution of the video.
+
+ *  @param onTouchEvent     A SDLOnTouchEvent with coordinates.
+ */
+- (SDLOnTouchEvent *)sdl_applyScaleToEventCoordinates:(SDLOnTouchEvent *)onTouchEvent {
+    float scale = self.videoStreamingCapability.scale.floatValue;
+    if (scale > 1) {
+        for (SDLTouchEvent *touchEvent in onTouchEvent.event) {
+            for (SDLTouchCoord *coord in touchEvent.coord) {
+                coord.x = @(coord.x.floatValue / scale);
+                coord.y = @(coord.y.floatValue / scale);
+            }
+        }
+    }
+    return onTouchEvent;
+}
+
 #pragma mark - Private
 /**
  *  Handles a BEGIN touch event sent by Core

SmartDeviceLink/SDLVideoStreamingCapability.h

@@ -18,7 +18,12 @@ NS_ASSUME_NONNULL_BEGIN
  */
 @interface SDLVideoStreamingCapability : SDLRPCStruct
 
-- (instancetype)initWithPreferredResolution:(nullable SDLImageResolution *)preferredResolution maxBitrate:(int32_t)maxBitrate supportedFormats:(nullable NSArray<SDLVideoStreamingFormat *> *)supportedFormats hapticDataSupported:(BOOL)hapticDataSupported;
+- (instancetype)initWithPreferredResolution:(nullable SDLImageResolution *)preferredResolution maxBitrate:(int32_t)maxBitrate supportedFormats:(nullable NSArray<SDLVideoStreamingFormat *> *)supportedFormats hapticDataSupported:(BOOL)hapticDataSupported __deprecated_msg("Use initWithPreferredResolution:maxBitrate:supportedFormats:hapticDataSupported:diagonalScreenSize:pixelPerInch:scale: instead");;
+
+/**
+ Contains information about this system's video streaming capabilities
+ */
+- (instancetype)initWithPreferredResolution:(nullable SDLImageResolution *)preferredResolution maxBitrate:(int32_t)maxBitrate supportedFormats:(nullable NSArray<SDLVideoStreamingFormat *> *)supportedFormats hapticDataSupported:(BOOL)hapticDataSupported diagonalScreenSize:(float)diagonalScreenSize pixelPerInch:(float)pixelPerInch scale:(float)scale;
 
 /**
  The preferred resolution of a video stream for decoding and rendering on HMI
@@ -48,6 +53,27 @@ NS_ASSUME_NONNULL_BEGIN
  */
 @property (nullable, strong, nonatomic) NSNumber<SDLBool> *hapticSpatialDataSupported;
 
+/**
+ The diagonal screen size in inches.
+
+ Optional
+ */
+@property (nullable, strong, nonatomic) NSNumber<SDLFloat> *diagonalScreenSize;
+
+/**
+ PPI is the diagonal resolution in pixels divided by the diagonal screen size in inches.
+
+ Optional
+ */
+@property (nullable, strong, nonatomic) NSNumber<SDLFloat> *pixelPerInch;
+
+/**
+ The scaling factor the app should use to change the size of the projecting view.
+
+ Optional
+ */
+@property (nullable, strong, nonatomic) NSNumber<SDLFloat> *scale;
+
 @end
 
 NS_ASSUME_NONNULL_END

SmartDeviceLink/SDLVideoStreamingCapability.m

@@ -18,6 +18,10 @@
 @implementation SDLVideoStreamingCapability
 
 - (instancetype)initWithPreferredResolution:(nullable SDLImageResolution *)preferredResolution maxBitrate:(int32_t)maxBitrate supportedFormats:(nullable NSArray<SDLVideoStreamingFormat *> *)supportedFormats hapticDataSupported:(BOOL)hapticDataSupported {
+    return [self initWithPreferredResolution:preferredResolution maxBitrate:maxBitrate supportedFormats:supportedFormats hapticDataSupported:hapticDataSupported diagonalScreenSize:0 pixelPerInch:0 scale:SDLVideoStreamingCapability.sdl_DefaultScale.floatValue];
+}
+
+- (instancetype)initWithPreferredResolution:(nullable SDLImageResolution *)preferredResolution maxBitrate:(int32_t)maxBitrate supportedFormats:(nullable NSArray<SDLVideoStreamingFormat *> *)supportedFormats hapticDataSupported:(BOOL)hapticDataSupported diagonalScreenSize:(float)diagonalScreenSize pixelPerInch:(float)pixelPerInch scale:(float)scale {
     self = [self init];
     if (!self) {
         return self;
@@ -27,6 +31,9 @@ - (instancetype)initWithPreferredResolution:(nullable SDLImageResolution *)prefe
     self.preferredResolution = preferredResolution;
     self.supportedFormats = supportedFormats;
     self.hapticSpatialDataSupported = @(hapticDataSupported);
+    self.diagonalScreenSize = @(diagonalScreenSize);
+    self.pixelPerInch = @(pixelPerInch);
+    self.scale = @(scale);
 
     return self;
 }
@@ -63,6 +70,38 @@ - (void)setHapticSpatialDataSupported:(nullable NSNumber<SDLBool> *)hapticSpatia
     return [self.store sdl_objectForName:SDLRPCParameterNameHapticSpatialDataSupported ofClass:NSNumber.class error:nil];
 }
 
+- (void)setDiagonalScreenSize:(nullable NSNumber<SDLFloat> *)diagonalScreenSize {
+    [self.store sdl_setObject:diagonalScreenSize forName:SDLRPCParameterNameDiagonalScreenSize];
+}
+
+- (nullable NSNumber<SDLFloat> *)diagonalScreenSize {
+    return [self.store sdl_objectForName:SDLRPCParameterNameDiagonalScreenSize ofClass:NSNumber.class error:nil];
+}
+
+- (void)setPixelPerInch:(nullable NSNumber<SDLFloat> *)pixelPerInch {
+    [self.store sdl_setObject:pixelPerInch forName:SDLRPCParameterNamePixelPerInch];
+}
+
+- (nullable NSNumber<SDLFloat> *)pixelPerInch {
+    return [self.store sdl_objectForName:SDLRPCParameterNamePixelPerInch ofClass:NSNumber.class error:nil];
+}
+
+- (void)setScale:(nullable NSNumber<SDLFloat> *)scale {
+    [self.store sdl_setObject:scale forName:SDLRPCParameterNameScale];
+}
+
+- (nullable NSNumber<SDLFloat> *)scale {
+    NSNumber<SDLFloat> *scale = [self.store sdl_objectForName:SDLRPCParameterNameScale ofClass:NSNumber.class error:nil];
+    if (scale != nil) {
+        return scale;
+    }
+    return SDLVideoStreamingCapability.sdl_DefaultScale;
+}
+
++ (NSNumber<SDLFloat> *)sdl_DefaultScale {
+    return @1.0;
+}
+
 @end
 
 NS_ASSUME_NONNULL_END

SmartDeviceLinkTests/DevAPISpecs/SDLStreamingVideoLifecycleManagerSpec.m

@@ -40,6 +40,7 @@
 @interface SDLStreamingVideoLifecycleManager ()
 @property (copy, nonatomic, readonly) NSString *appName;
 @property (copy, nonatomic, readonly) NSString *videoStreamBackgroundString;
+@property (strong, nonatomic, readwrite) SDLVideoStreamingCapability *videoStreamingCapability;
 @end
 
 QuickSpecBegin(SDLStreamingVideoLifecycleManagerSpec)
@@ -411,6 +412,14 @@ __block void (^sendNotificationForHMILevel)(SDLHMILevel hmiLevel, SDLVideoStream
         });
 
         describe(@"sending a video capabilities request", ^{
+            __block SDLImageResolution *resolution = [[SDLImageResolution alloc] initWithWidth:42 height:69];
+            __block int32_t maxBitrate = 12345;
+            __block NSArray<SDLVideoStreamingFormat *> *testFormats = @[[[SDLVideoStreamingFormat alloc] initWithCodec:SDLVideoStreamingCodecH265 protocol:SDLVideoStreamingProtocolRTMP], [[SDLVideoStreamingFormat alloc] initWithCodec:SDLVideoStreamingCodecH264 protocol:SDLVideoStreamingProtocolRTP]];
+            __block BOOL testHapticsSupported = YES;
+            __block float diagonalScreenSize = 22.0;
+            __block float pixelPerInch = 96.0;
+            __block float scale = 1.0;
+
             beforeEach(^{
                 [streamingLifecycleManager.videoStreamStateMachine setToState:SDLVideoStreamManagerStateStarting fromOldState:nil callEnterTransition:YES];
             });
@@ -443,22 +452,13 @@ __block void (^sendNotificationForHMILevel)(SDLHMILevel hmiLevel, SDLVideoStream
                     });
 
                     context(@"and receiving a response", ^{
-                        __block SDLImageResolution *resolution = nil;
-                        __block int32_t maxBitrate = 0;
-                        __block NSArray<SDLVideoStreamingFormat *> *testFormats = nil;
-                        __block BOOL testHapticsSupported = NO;
-
                         beforeEach(^{
                             SDLGetSystemCapabilityResponse *response = [[SDLGetSystemCapabilityResponse alloc] init];
                             response.success = @YES;
                             response.systemCapability = [[SDLSystemCapability alloc] init];
                             response.systemCapability.systemCapabilityType = SDLSystemCapabilityTypeVideoStreaming;
-
-                            resolution = [[SDLImageResolution alloc] initWithWidth:42 height:69];
-                            maxBitrate = 12345;
-                            testFormats = @[[[SDLVideoStreamingFormat alloc] initWithCodec:SDLVideoStreamingCodecH265 protocol:SDLVideoStreamingProtocolRTMP], [[SDLVideoStreamingFormat alloc] initWithCodec:SDLVideoStreamingCodecH264 protocol:SDLVideoStreamingProtocolRTP]];
-                            testHapticsSupported = YES;
-                            response.systemCapability.videoStreamingCapability = [[SDLVideoStreamingCapability alloc] initWithPreferredResolution:resolution maxBitrate:maxBitrate supportedFormats:testFormats hapticDataSupported:testHapticsSupported];
+
+                            response.systemCapability.videoStreamingCapability = [[SDLVideoStreamingCapability alloc] initWithPreferredResolution:resolution maxBitrate:maxBitrate supportedFormats:testFormats hapticDataSupported:testHapticsSupported diagonalScreenSize:diagonalScreenSize pixelPerInch:pixelPerInch scale:scale];
                             [testConnectionManager respondToLastRequestWithResponse:response];
                         });
 
@@ -485,6 +485,12 @@ __block void (^sendNotificationForHMILevel)(SDLHMILevel hmiLevel, SDLVideoStream
                             expect(preferredResolution.resolutionHeight).to(equal(@69));
                             expect(preferredResolution.resolutionWidth).to(equal(@42));
                         });
+
+                        it(@"should have correct video streaming capability values", ^{
+                            expect(streamingLifecycleManager.videoStreamingCapability.diagonalScreenSize).to(equal(22.0));
+                            expect(streamingLifecycleManager.videoStreamingCapability.pixelPerInch).to(equal(96.0));
+                            expect(streamingLifecycleManager.videoStreamingCapability.scale).to(equal(1.0));
+                        });
                     });
                 });
             });
@@ -507,6 +513,8 @@ __block void (^sendNotificationForHMILevel)(SDLHMILevel hmiLevel, SDLVideoStream
                     testVideoHeader.frameData = SDLFrameInfoStartServiceACK;
                     testVideoHeader.encrypted = YES;
                     testVideoHeader.serviceType = SDLServiceTypeVideo;
+
+                    streamingLifecycleManager.videoStreamingCapability = [[SDLVideoStreamingCapability alloc] initWithPreferredResolution:resolution maxBitrate:maxBitrate supportedFormats:testFormats hapticDataSupported:testHapticsSupported diagonalScreenSize:diagonalScreenSize pixelPerInch:pixelPerInch scale:scale];
                 });
 
                 context(@"with data", ^{