Implement SDL 0179 - Pixel density and Scale

SmartDeviceLink/SDLCarWindow.h

@@ -24,7 +24,20 @@ NS_ASSUME_NONNULL_BEGIN
  @param configuration The streaming media configuration
  @return An instance of this class
  */
-- (instancetype)initWithStreamManager:(SDLStreamingVideoLifecycleManager *)streamManager configuration:(SDLStreamingMediaConfiguration *)configuration;
+- (instancetype)initWithStreamManager:(SDLStreamingVideoLifecycleManager *)streamManager
+                        configuration:(SDLStreamingMediaConfiguration *)configuration;
+
+/**
+ Initialize the CarWindow automatic streamer.
+
+ @param streamManager The stream manager to use for retrieving head unit dimension details and forwarding video frame data
+ @param configuration The streaming media configuration
+ @param scale The scale factor value to scale coordinates from one coordinate space to another
+ @return An instance of this class
+ */
+- (instancetype)initWithStreamManager:(SDLStreamingVideoLifecycleManager *)streamManager
+                        configuration:(nonnull SDLStreamingMediaConfiguration *)configuration
+                                scale:(float)scale;
 
 /**
  *  View Controller that will be streamed.

SmartDeviceLink/SDLCarWindow.m

@@ -10,6 +10,7 @@
 #import <CommonCrypto/CommonDigest.h>
 #import <ImageIO/ImageIO.h>
 #import <MobileCoreServices/MobileCoreServices.h>
+#import <simd/simd.h>
 
 #import "SDLCarWindow.h"
 #import "SDLGlobals.h"
@@ -21,6 +22,7 @@
 #import "SDLStreamingMediaConfiguration.h"
 #import "SDLStreamingVideoLifecycleManager.h"
 #import "SDLStreamingMediaManagerConstants.h"
+#import "SDLVideoStreamingCapability.h"
 
 NS_ASSUME_NONNULL_BEGIN
 
@@ -37,14 +39,24 @@ @interface SDLCarWindow ()
 
 @property (assign, nonatomic, getter=isVideoStreamStarted) BOOL videoStreamStarted;
 
+@property (assign, nonatomic) float sdl_scale;
+
 @end
 
 @implementation SDLCarWindow
 
-- (instancetype)initWithStreamManager:(SDLStreamingVideoLifecycleManager *)streamManager configuration:(nonnull SDLStreamingMediaConfiguration *)configuration {
+- (instancetype)initWithStreamManager:(SDLStreamingVideoLifecycleManager *)streamManager
+                        configuration:(nonnull SDLStreamingMediaConfiguration *)configuration {
+    return [self initWithStreamManager:streamManager configuration:configuration scale:1.f];
+}
+
+- (instancetype)initWithStreamManager:(SDLStreamingVideoLifecycleManager *)streamManager
+                        configuration:(nonnull SDLStreamingMediaConfiguration *)configuration
+                                scale:(float)scale {
     self = [super init];
     if (!self) { return nil; }
 
+    _sdl_scale = simd_clamp(scale, 1.f, 10.f);
     _streamManager = streamManager;
     _renderingType = configuration.carWindowRenderingType;
     _allowMultipleOrientations = configuration.allowMultipleViewControllerOrientations;
@@ -71,7 +83,7 @@ - (void)syncFrame {
         return;
     }
 
-    CGRect bounds = self.rootViewController.view.bounds;
+    CGRect bounds = self.sdl_getScaledScreenSizeFrame;
 
     UIGraphicsBeginImageContextWithOptions(bounds.size, YES, 1.0f);
     switch (self.renderingType) {
@@ -120,13 +132,17 @@ - (void)sdl_didReceiveVideoStreamStarted:(NSNotification *)notification {
 
     dispatch_async(dispatch_get_main_queue(), ^{
         // If the video stream has started, we want to resize the streamingViewController to the size from the RegisterAppInterface
-        self.rootViewController.view.frame = CGRectMake(0, 0, self.streamManager.screenSize.width, self.streamManager.screenSize.height);
+        self.rootViewController.view.frame = self.sdl_getScaledScreenSizeFrame;
         self.rootViewController.view.bounds = self.rootViewController.view.frame;
-
+        
         SDLLogD(@"Video stream started, setting CarWindow frame to: %@", NSStringFromCGRect(self.rootViewController.view.bounds));
     });
 }
 
+- (CGRect)sdl_getScaledScreenSizeFrame {
+    return CGRectMake(0, 0, self.streamManager.screenSize.width / self.sdl_scale, self.streamManager.screenSize.height / self.sdl_scale);
+}
+
 - (void)sdl_didReceiveVideoStreamStopped:(NSNotification *)notification {
     self.videoStreamStarted = false;
 

SmartDeviceLink/SDLRPCParameterNames.h

@@ -146,6 +146,7 @@ extern SDLRPCParameterName const SDLRPCParameterNameDestinationETA;
 extern SDLRPCParameterName const SDLRPCParameterNameDetails;
 extern SDLRPCParameterName const SDLRPCParameterNameDeviceInfo;
 extern SDLRPCParameterName const SDLRPCParameterNameDeviceStatus;
+extern SDLRPCParameterName const SDLRPCParameterNameDiagonalScreenSize;
 extern SDLRPCParameterName const SDLRPCParameterNameDialNumberEnabled;
 extern SDLRPCParameterName const SDLRPCParameterNameDIDLocation;
 extern SDLRPCParameterName const SDLRPCParameterNameDIDResult;
@@ -460,6 +461,7 @@ extern SDLRPCParameterName const SDLRPCParameterNamePhoneCall;
 extern SDLRPCParameterName const SDLRPCParameterNamePhoneCapability;
 extern SDLRPCParameterName const SDLRPCParameterNamePhoneNumber;
 extern SDLRPCParameterName const SDLRPCParameterNamePhoneRoaming;
+extern SDLRPCParameterName const SDLRPCParameterNamePixelPerInch;
 extern SDLRPCParameterName const SDLRPCParameterNamePlaylistName;
 extern SDLRPCParameterName const SDLRPCParameterNamePlayTone;
 extern SDLRPCParameterName const SDLRPCParameterNamePosition;
@@ -534,6 +536,7 @@ extern SDLRPCParameterName const SDLRPCParameterNameRPM;
 extern SDLRPCParameterName const SDLRPCParameterNameRadioText;
 extern SDLRPCParameterName const SDLRPCParameterNameSamplingRate;
 extern SDLRPCParameterName const SDLRPCParameterNameSatellites;
+extern SDLRPCParameterName const SDLRPCParameterNameScale;
 extern SDLRPCParameterName const SDLRPCParameterNameScreenParams;
 extern SDLRPCParameterName const SDLRPCParameterNameScrollableMessageBody;
 extern SDLRPCParameterName const SDLRPCParameterNameSDLVersion;

SmartDeviceLink/SDLRPCParameterNames.m

@@ -143,6 +143,7 @@
 SDLRPCParameterName const SDLRPCParameterNameDeviceInfo = @"deviceInfo";
 SDLRPCParameterName const SDLRPCParameterNameDeviceStatus = @"deviceStatus";
 SDLRPCParameterName const SDLRPCParameterNameDetails = @"details";
+SDLRPCParameterName const SDLRPCParameterNameDiagonalScreenSize = @"diagonalScreenSize";
 SDLRPCParameterName const SDLRPCParameterNameDialNumberEnabled = @"dialNumberEnabled";
 SDLRPCParameterName const SDLRPCParameterNameDIDLocation = @"didLocation";
 SDLRPCParameterName const SDLRPCParameterNameDIDResult = @"didResult";
@@ -454,6 +455,7 @@
 SDLRPCParameterName const SDLRPCParameterNamePhoneCapability = @"phoneCapability";
 SDLRPCParameterName const SDLRPCParameterNamePhoneNumber = @"phoneNumber";
 SDLRPCParameterName const SDLRPCParameterNamePhoneRoaming = @"phoneRoaming";
+SDLRPCParameterName const SDLRPCParameterNamePixelPerInch = @"pixelPerInch";
 SDLRPCParameterName const SDLRPCParameterNamePrimaryColor = @"primaryColor";
 SDLRPCParameterName const SDLRPCParameterNamePlaylistName = @"playlistName";
 SDLRPCParameterName const SDLRPCParameterNamePlayTone = @"playTone";
@@ -529,6 +531,7 @@
 SDLRPCParameterName const SDLRPCParameterNameRadioText  = @"RT";
 SDLRPCParameterName const SDLRPCParameterNameSamplingRate = @"samplingRate";
 SDLRPCParameterName const SDLRPCParameterNameSatellites = @"satellites";
+SDLRPCParameterName const SDLRPCParameterNameScale = @"scale";
 SDLRPCParameterName const SDLRPCParameterNameScreenParams = @"screenParams";
 SDLRPCParameterName const SDLRPCParameterNameScrollableMessageBody = @"scrollableMessageBody";
 SDLRPCParameterName const SDLRPCParameterNameSDLVersion = @"sdlVersion";

SmartDeviceLink/SDLStreamingVideoLifecycleManager.h

@@ -22,6 +22,7 @@
 @class SDLStateMachine;
 @class SDLStreamingMediaConfiguration;
 @class SDLTouchManager;
+@class SDLVideoStreamingCapability;
 
 @protocol SDLConnectionManagerType;
 @protocol SDLFocusableItemLocatorType;
@@ -35,6 +36,7 @@ NS_ASSUME_NONNULL_BEGIN
 
 @property (strong, nonatomic, readonly) SDLStateMachine *videoStreamStateMachine;
 @property (strong, nonatomic, readonly) SDLVideoStreamManagerState *currentVideoStreamState;
+@property (nullable, strong, nonatomic, readonly) SDLVideoStreamingCapability *videoStreamingCapability;
 
 @property (strong, nonatomic, readonly) SDLStateMachine *appStateMachine;
 @property (strong, nonatomic, readonly) SDLAppState *currentAppState;

SmartDeviceLink/SDLStreamingVideoLifecycleManager.m

@@ -59,6 +59,8 @@ @interface SDLStreamingVideoLifecycleManager() <SDLVideoEncoderDelegate>
 @property (assign, nonatomic, readonly, getter=isAppStateVideoStreamCapable) BOOL appStateVideoStreamCapable;
 @property (assign, nonatomic, readonly, getter=isHmiStateVideoStreamCapable) BOOL hmiStateVideoStreamCapable;
 
+@property (nullable, strong, nonatomic, readwrite) SDLVideoStreamingCapability *videoStreamingCapability;
+
 @property (strong, nonatomic, readwrite) SDLStateMachine *videoStreamStateMachine;
 @property (strong, nonatomic, readwrite) SDLStateMachine *appStateMachine;
 
@@ -112,11 +114,11 @@ - (instancetype)initWithConnectionManager:(id<SDLConnectionManagerType>)connecti
         }
 
         SDLLogD(@"Initializing CarWindow");
-        _carWindow = [[SDLCarWindow alloc] initWithStreamManager:self configuration:configuration.streamingMediaConfig];
+        _carWindow = [[SDLCarWindow alloc] initWithStreamManager:self configuration:configuration.streamingMediaConfig scale:self.sdl_scale];
         _carWindow.rootViewController = configuration.streamingMediaConfig.rootViewController;
     }
 
-    _touchManager = [[SDLTouchManager alloc] initWithHitTester:(id)_focusableItemManager];
+    _touchManager = [[SDLTouchManager alloc] initWithHitTester:(id)_focusableItemManager scale:self.sdl_scale];
 
     _requestedEncryptionType = configuration.streamingMediaConfig.maximumDesiredEncryption;
     _dataSource = configuration.streamingMediaConfig.dataSource;
@@ -252,6 +254,11 @@ - (SDLVideoStreamManagerState *)currentVideoStreamState {
     return self.videoStreamStateMachine.currentState;
 }
 
+- (float)sdl_scale {
+    const float scale = self.videoStreamingCapability.scale.floatValue;
+    return (scale > 1.0) ? scale : 1.0;
+}
+
 #pragma mark - State Machines
 #pragma mark App State
 + (NSDictionary<SDLState *, SDLAllowableStateTransitions *> *)sdl_appStateTransitionDictionary {
@@ -403,7 +410,9 @@ - (void)didEnterStateVideoStreamReady {
         NSAssert(self.videoFormat != nil, @"No video format is known, but it must be if we got a protocol start service response");
 
         SDLLogD(@"Attempting to create video encoder");
-        self.videoEncoder = [[SDLH264VideoEncoder alloc] initWithProtocol:self.videoFormat.protocol dimensions:self.screenSize ssrc:self.ssrc properties:self.videoEncoderSettings delegate:self error:&error];
+        const float scale = self.sdl_scale;
+        CGSize scaledScreenSize = CGSizeMake(self.screenSize.width / scale, self.screenSize.height / scale);
+        self.videoEncoder = [[SDLH264VideoEncoder alloc] initWithProtocol:self.videoFormat.protocol dimensions:scaledScreenSize ssrc:self.ssrc properties:self.videoEncoderSettings delegate:self error:&error];
 
         if (error || self.videoEncoder == nil) {
             SDLLogE(@"Could not create a video encoder: %@", error);
@@ -728,6 +737,8 @@ - (void)sdl_requestVideoCapabilities:(SDLVideoCapabilityResponseHandler)response
         }
 
         SDLVideoStreamingCapability *videoCapability = ((SDLGetSystemCapabilityResponse *)response).systemCapability.videoStreamingCapability;
+        self.videoStreamingCapability = videoCapability;
+        self.touchManager.scale = self.sdl_scale;
         SDLLogD(@"Video capabilities response received: %@", videoCapability);
         responseHandler(videoCapability);
     }];

SmartDeviceLink/SDLTouchManager.h

@@ -78,6 +78,12 @@ typedef void(^SDLTouchEventHandler)(SDLTouch *touch, SDLTouchType type);
  */
 @property (nonatomic, assign, getter=isTouchEnabled) BOOL touchEnabled;
 
+/**
+ *  @abstract
+ The scale factor value to scale coordinates from one coordinate space to another
+ */
+@property (nonatomic, assign) float scale;
+
 /**
  *  @abstract
  *      Cancels pending touch event timers that may be in progress.
@@ -96,6 +102,15 @@ typedef void(^SDLTouchEventHandler)(SDLTouch *touch, SDLTouchType type);
  */
 - (instancetype)initWithHitTester:(nullable id<SDLFocusableItemHitTester>)hitTester;
 
+/**
+ Initialize a touch manager with a hit tester if available and a scale factor
+
+ @param hitTester The hit tester to be used to correlate a point with a view
+ @param scale The scale factor value to scale coordinates from one coordinate space to another
+ @return The initialized touch manager
+ */
+- (instancetype)initWithHitTester:(nullable id<SDLFocusableItemHitTester>)hitTester scale:(float)scale;
+
 /**
  Called by SDLStreamingMediaManager in sync with the streaming framerate. This helps to moderate panning gestures by allowing the UI to be modified in time with the framerate.
  */

SmartDeviceLink/SDLTouchManager.m

@@ -6,8 +6,9 @@
 //  Copyright © 2016 smartdevicelink. All rights reserved.
 //
 
-#import "SDLTouchManager.h"
+#import <simd/simd.h>
 
+#import "SDLTouchManager.h"
 #import "CGPoint_Util.h"
 
 #import "SDLGlobals.h"
@@ -102,25 +103,31 @@ @interface SDLTouchManager ()
 
 @implementation SDLTouchManager
 
-- (instancetype)initWithHitTester:(nullable id<SDLFocusableItemHitTester>)hitTester {
-    self = [super init];
-    if (!self) {
+- (instancetype)initWithHitTester:(nullable id<SDLFocusableItemHitTester>)hitTester
+                            scale:(float)scale {
+    if (!(self = [super init])) {
         return nil;
     }
-
+    
     _hitTester = hitTester;
+    _scale = simd_clamp(scale, 1.f, 10.f);
     _movementTimeThreshold = 0.05f;
     _tapTimeThreshold = 0.4f;
     _tapDistanceThreshold = 50.0f;
     _panDistanceThreshold = 8.0f;
     _touchEnabled = YES;
     _enableSyncedPanning = YES;
 
+    //TODO: unsubscribe from notifications somewhere
     [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(sdl_onTouchEvent:) name:SDLDidReceiveTouchEventNotification object:nil];
 
     return self;
 }
 
+- (instancetype)initWithHitTester:(nullable id<SDLFocusableItemHitTester>)hitTester {
+    return [self initWithHitTester:hitTester scale:1.f];
+}
+
 #pragma mark - Public
 - (void)cancelPendingTouches {
     [self sdl_cancelSingleTapTimer];
@@ -184,7 +191,8 @@ - (void)sdl_onTouchEvent:(SDLRPCNotificationNotification *)notification {
         return;
     }
 
-    SDLOnTouchEvent* onTouchEvent = (SDLOnTouchEvent*)notification.notification;
+    SDLOnTouchEvent *onTouchEvent = (SDLOnTouchEvent *)notification.notification;
+    onTouchEvent = [self sdl_applyScaleToEventCoordinates:onTouchEvent.copy];
 
     SDLTouchType touchType = onTouchEvent.type;
     [onTouchEvent.event enumerateObjectsUsingBlock:^(SDLTouchEvent *touchEvent, NSUInteger idx, BOOL *stop) {
@@ -212,6 +220,27 @@ - (void)sdl_onTouchEvent:(SDLRPCNotificationNotification *)notification {
     }];
 }
 
+/**
+ *  Modifies the existing coordinates of the SDLOnTouchEvent, based on the received 'scale' value.
+
+ *  This will match the coordinates to the scaled resolution of the video.
+
+ *  @param onTouchEvent     A SDLOnTouchEvent with coordinates.
+ */
+- (SDLOnTouchEvent *)sdl_applyScaleToEventCoordinates:(SDLOnTouchEvent *)onTouchEvent {
+    const float scale = self.scale;
+    if (scale <= 1.f) {
+        return onTouchEvent;
+    }
+    for (SDLTouchEvent *touchEvent in onTouchEvent.event) {
+        for (SDLTouchCoord *coord in touchEvent.coord) {
+            coord.x = @(coord.x.floatValue / scale);
+            coord.y = @(coord.y.floatValue / scale);
+        }
+    }
+    return onTouchEvent;
+}
+
 #pragma mark - Private
 /**
  *  Handles a BEGIN touch event sent by Core

SmartDeviceLink/SDLVideoStreamingCapability.h

@@ -18,7 +18,12 @@ NS_ASSUME_NONNULL_BEGIN
  */
 @interface SDLVideoStreamingCapability : SDLRPCStruct
 
-- (instancetype)initWithPreferredResolution:(nullable SDLImageResolution *)preferredResolution maxBitrate:(int32_t)maxBitrate supportedFormats:(nullable NSArray<SDLVideoStreamingFormat *> *)supportedFormats hapticDataSupported:(BOOL)hapticDataSupported;
+- (instancetype)initWithPreferredResolution:(nullable SDLImageResolution *)preferredResolution maxBitrate:(int32_t)maxBitrate supportedFormats:(nullable NSArray<SDLVideoStreamingFormat *> *)supportedFormats hapticDataSupported:(BOOL)hapticDataSupported __deprecated_msg("Use initWithPreferredResolution:maxBitrate:supportedFormats:hapticDataSupported:diagonalScreenSize:pixelPerInch:scale: instead");;
+
+/**
+ Contains information about this system's video streaming capabilities
+ */
+- (instancetype)initWithPreferredResolution:(nullable SDLImageResolution *)preferredResolution maxBitrate:(int32_t)maxBitrate supportedFormats:(nullable NSArray<SDLVideoStreamingFormat *> *)supportedFormats hapticDataSupported:(BOOL)hapticDataSupported diagonalScreenSize:(float)diagonalScreenSize pixelPerInch:(float)pixelPerInch scale:(float)scale;
 
 /**
  The preferred resolution of a video stream for decoding and rendering on HMI
@@ -48,6 +53,27 @@ NS_ASSUME_NONNULL_BEGIN
  */
 @property (nullable, strong, nonatomic) NSNumber<SDLBool> *hapticSpatialDataSupported;
 
+/**
+ The diagonal screen size in inches.
+
+ Optional
+ */
+@property (nullable, strong, nonatomic) NSNumber<SDLFloat> *diagonalScreenSize;
+
+/**
+ PPI is the diagonal resolution in pixels divided by the diagonal screen size in inches.
+
+ Optional
+ */
+@property (nullable, strong, nonatomic) NSNumber<SDLFloat> *pixelPerInch;
+
+/**
+ The scaling factor the app should use to change the size of the projecting view.
+
+ Optional
+ */
+@property (nullable, strong, nonatomic) NSNumber<SDLFloat> *scale;
+
 @end
 
 NS_ASSUME_NONNULL_END