Fixed video/audio end services sometimes not being sent when secondary transport shuts down

SmartDeviceLink-iOS.xcodeproj/project.pbxproj

@@ -1395,6 +1395,7 @@
 		888DBAF022D528DE002A0AE2 /* SDLCloseApplicationResponse.m in Sources */ = {isa = PBXBuildFile; fileRef = 888DBAEE22D528DE002A0AE2 /* SDLCloseApplicationResponse.m */; };
 		888F86FE221DEE200052FE4C /* SDLAsynchronousRPCOperation.m in Sources */ = {isa = PBXBuildFile; fileRef = 888F86FD221DEE1F0052FE4C /* SDLAsynchronousRPCOperation.m */; };
 		888F8700221DF4880052FE4C /* SDLAsynchronousRPCOperationSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = 888F86FF221DF4880052FE4C /* SDLAsynchronousRPCOperationSpec.m */; };
+		88A118272412ACAD00B45FB5 /* SDLStreamingMediaManagerSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = DA661E2B1E553E7E001C1345 /* SDLStreamingMediaManagerSpec.m */; };
 		88A1CF1E21669AC7001ACC75 /* SDLLifecycleConfigurationUpdateSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = 88A1CF1D21669AC7001ACC75 /* SDLLifecycleConfigurationUpdateSpec.m */; };
 		88A4A0FA22242AB400C6F01D /* SDLNavigationServiceDataSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = 88A4A0F922242AB400C6F01D /* SDLNavigationServiceDataSpec.m */; };
 		88A5E7F4220B57F900495E8A /* SDLOnSystemCapabilityUpdatedSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = 88A5E7F3220B57F900495E8A /* SDLOnSystemCapabilityUpdatedSpec.m */; };
@@ -1606,7 +1607,6 @@
 		E9C32B9E1AB20C5900F283AF /* EAAccessoryManager+SDLProtocols.h in Headers */ = {isa = PBXBuildFile; fileRef = E9C32B9A1AB20C5900F283AF /* EAAccessoryManager+SDLProtocols.h */; };
 		E9C32B9F1AB20C5900F283AF /* EAAccessoryManager+SDLProtocols.m in Sources */ = {isa = PBXBuildFile; fileRef = E9C32B9B1AB20C5900F283AF /* EAAccessoryManager+SDLProtocols.m */; };
 		EE28F9E3209802A500B1B61D /* SDLStreamingProtocolDelegate.h in Headers */ = {isa = PBXBuildFile; fileRef = EE6CBF872064CAEE00EEE0CA /* SDLStreamingProtocolDelegate.h */; };
-		EE38C0C3211C440400E170AD /* SDLSecondaryTransportPrimaryProtocolHandler.m in Sources */ = {isa = PBXBuildFile; fileRef = EE38C0C2211C440400E170AD /* SDLSecondaryTransportPrimaryProtocolHandler.m */; };
 		EE460E082066B5F20006EDD3 /* SDLControlFramePayloadTransportEventUpdateSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = EE460E072066B5F20006EDD3 /* SDLControlFramePayloadTransportEventUpdateSpec.m */; };
 		EE460E0A2066B6E40006EDD3 /* SDLControlFramePayloadRegisterSecondaryTransportNakSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = EE460E092066B6E40006EDD3 /* SDLControlFramePayloadRegisterSecondaryTransportNakSpec.m */; };
 		EE5D1B33208EBCA900D17216 /* SDLTCPTransportSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = EE5D1B32208EBCA900D17216 /* SDLTCPTransportSpec.m */; };
@@ -3352,8 +3352,6 @@
 		E9C32B991AB20C5900F283AF /* EAAccessory+SDLProtocols.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "EAAccessory+SDLProtocols.m"; sourceTree = "<group>"; };
 		E9C32B9A1AB20C5900F283AF /* EAAccessoryManager+SDLProtocols.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "EAAccessoryManager+SDLProtocols.h"; sourceTree = "<group>"; };
 		E9C32B9B1AB20C5900F283AF /* EAAccessoryManager+SDLProtocols.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "EAAccessoryManager+SDLProtocols.m"; sourceTree = "<group>"; };
-		EE38C0C1211C43E100E170AD /* SDLSecondaryTransportPrimaryProtocolHandler.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SDLSecondaryTransportPrimaryProtocolHandler.h; sourceTree = "<group>"; };
-		EE38C0C2211C440400E170AD /* SDLSecondaryTransportPrimaryProtocolHandler.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SDLSecondaryTransportPrimaryProtocolHandler.m; sourceTree = "<group>"; };
 		EE460E072066B5F20006EDD3 /* SDLControlFramePayloadTransportEventUpdateSpec.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = SDLControlFramePayloadTransportEventUpdateSpec.m; path = ControlFramePayloadSpecs/SDLControlFramePayloadTransportEventUpdateSpec.m; sourceTree = "<group>"; };
 		EE460E092066B6E40006EDD3 /* SDLControlFramePayloadRegisterSecondaryTransportNakSpec.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; name = SDLControlFramePayloadRegisterSecondaryTransportNakSpec.m; path = ControlFramePayloadSpecs/SDLControlFramePayloadRegisterSecondaryTransportNakSpec.m; sourceTree = "<group>"; };
 		EE5D1B32208EBCA900D17216 /* SDLTCPTransportSpec.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SDLTCPTransportSpec.m; sourceTree = "<group>"; };
@@ -6513,8 +6511,6 @@
 			children = (
 				EE798CA32056120F008EDE8E /* SDLSecondaryTransportManager.h */,
 				EE798CA520561217008EDE8E /* SDLSecondaryTransportManager.m */,
-				EE38C0C1211C43E100E170AD /* SDLSecondaryTransportPrimaryProtocolHandler.h */,
-				EE38C0C2211C440400E170AD /* SDLSecondaryTransportPrimaryProtocolHandler.m */,
 			);
 			name = "Secondary Transport";
 			sourceTree = "<group>";
@@ -7700,7 +7696,6 @@
 				EED5CA021F4D18EC00F04000 /* SDLRAWH264Packetizer.m in Sources */,
 				9FE2471622D77AD500F8D2FC /* SDLWindowType.m in Sources */,
 				5D61FC851A84238C00846EE7 /* SDLDeviceLevelStatus.m in Sources */,
-				EE38C0C3211C440400E170AD /* SDLSecondaryTransportPrimaryProtocolHandler.m in Sources */,
 				5D9FDA981F2A7D3F00A495C8 /* emhashmap.c in Sources */,
 				5D61FD1E1A84238C00846EE7 /* SDLOnTBTClientState.m in Sources */,
 				5D0C29FD20D93D8C008B56CD /* SDLVideoStreamingState.m in Sources */,
@@ -8104,6 +8099,7 @@
 				162E83371A9BDE8B00906325 /* SDLResetGlobalPropertiesSpec.m in Sources */,
 				162E82DF1A9BDE8B00906325 /* SDLGlobalProperySpec.m in Sources */,
 				88DF998F22035D1700477AC1 /* SDLIAPSessionSpec.m in Sources */,
+				88A118272412ACAD00B45FB5 /* SDLStreamingMediaManagerSpec.m in Sources */,
 				5DD8406520FCE21A0082CE04 /* SDLElectronicParkBrakeStatusSpec.m in Sources */,
 				162E82F61A9BDE8B00906325 /* SDLRequestTypeSpec.m in Sources */,
 				5DE35E4520CAFC5D0034BE5A /* SDLChoiceCellSpec.m in Sources */,

SmartDeviceLink.podspec

@@ -326,7 +326,6 @@ sdefault.public_header_files = [
 'SmartDeviceLink/SDLSeatLocationCapability.h',
 'SmartDeviceLink/SDLSeatMemoryAction.h',
 'SmartDeviceLink/SDLSeatMemoryActionType.h',
-'SmartDeviceLink/SDLSupportedSeat.h',
 'SmartDeviceLink/SDLSecurityType.h',
 'SmartDeviceLink/SDLSendHapticData.h',
 'SmartDeviceLink/SDLSendHapticDataResponse.h',
@@ -378,6 +377,7 @@ sdefault.public_header_files = [
 'SmartDeviceLink/SDLSubscribeVehicleDataResponse.h',
 'SmartDeviceLink/SDLSubscribeWaypoints.h',
 'SmartDeviceLink/SDLSubscribeWaypointsResponse.h',
+'SmartDeviceLink/SDLSupportedSeat.h',
 'SmartDeviceLink/SDLSyncMsgVersion.h',
 'SmartDeviceLink/SDLMsgVersion.h',
 'SmartDeviceLink/SDLSyncPData.h',

SmartDeviceLink/SDLCarWindow.m

@@ -92,8 +92,14 @@ - (void)syncFrame {
     CGImageRef imageRef = screenshot.CGImage;
     CVPixelBufferRef pixelBuffer = [self.class sdl_pixelBufferForImageRef:imageRef usingPool:self.streamManager.pixelBufferPool];
     if (pixelBuffer != nil) {
-        [self.streamManager sendVideoData:pixelBuffer];
+        BOOL success = [self.streamManager sendVideoData:pixelBuffer];
+        if (!success) {
+            SDLLogE(@"Video frame will not be sent because the video frame encoding failed");
+            return;
+        }
         CVPixelBufferRelease(pixelBuffer);
+    } else {
+        SDLLogE(@"Video frame will not be sent because the pixelBuffer is nil");
     }
 }
 
@@ -191,6 +197,11 @@ - (void)sdl_applyDisplayDimensionsToRootViewController:(UIViewController *)rootV
         return;
     }
 
+    if (CGRectEqualToRect(rootViewController.view.frame, self.streamManager.videoScaleManager.appViewportFrame)) {
+        SDLLogV(@"The rootViewController frame is already the correct size: %@", NSStringFromCGRect(rootViewController.view.frame));
+        return;
+    }
+
     rootViewController.view.frame = self.streamManager.videoScaleManager.appViewportFrame;
     rootViewController.view.bounds = rootViewController.view.frame;
 

SmartDeviceLink/SDLH264VideoEncoder.h

@@ -51,7 +51,7 @@ extern NSString *const SDLErrorDomainVideoEncoder;
 /**
  *  The pixel buffer pool reference returned back from an active VTCompressionSessionRef encoder.
  *
- *  @warning This will only return a valid pixel buffer pool after the encoder has been initialized (when the video     session has started).
+ *  @warning This will only return a valid pixel buffer pool after the encoder has been initialized (when the video session has started).
  *  @discussion Clients may call this once and retain the resulting pool, this call is cheap enough that it's OK to call it once per frame.
  */
 @property (assign, nonatomic, readonly) CVPixelBufferPoolRef CV_NULLABLE pixelBufferPool;

SmartDeviceLink/SDLH264VideoEncoder.m

@@ -26,6 +26,11 @@ @interface SDLH264VideoEncoder ()
 @property (assign, nonatomic) NSUInteger currentFrameNumber;
 @property (assign, nonatomic) double timestampOffset;
 
+@property (assign, nonatomic, readwrite) CVPixelBufferPoolRef CV_NULLABLE pixelBufferPool;
+
+/// Width and height of the video frame.
+@property (assign, nonatomic) CGSize dimensions;
+
 @end
 
 
@@ -54,6 +59,7 @@ - (instancetype)initWithProtocol:(SDLVideoStreamingProtocol)protocol dimensions:
     _compressionSession = NULL;
     _currentFrameNumber = 0;
     _videoEncoderSettings = properties;
+    _dimensions = dimensions;
 
     _delegate = delegate;
 
@@ -63,9 +69,8 @@ - (instancetype)initWithProtocol:(SDLVideoStreamingProtocol)protocol dimensions:
     status = VTCompressionSessionCreate(NULL, (int32_t)dimensions.width, (int32_t)dimensions.height, kCMVideoCodecType_H264, NULL, self.sdl_pixelBufferOptions, NULL, &sdl_videoEncoderOutputCallback, (__bridge void *)self, &_compressionSession);
 
     if (status != noErr) {
-        if (!*error) {
-            *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorConfigurationCompressionSessionCreationFailure userInfo:@{ @"OSStatus": @(status) }];
-            SDLLogE(@"Error attempting to create video compression session: %@", *error);
+        if (error != NULL) {
+            *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorConfigurationCompressionSessionCreationFailure userInfo:@{@"OSStatus":@(status), NSLocalizedDescriptionKey:@"Compression session could not be created"}];
         }
 
         return nil;
@@ -78,8 +83,8 @@ - (instancetype)initWithProtocol:(SDLVideoStreamingProtocol)protocol dimensions:
     CFDictionaryRef supportedProperties;
     status = VTSessionCopySupportedPropertyDictionary(self.compressionSession, &supportedProperties);
     if (status != noErr) {
-        if (!*error) {
-            *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{ @"OSStatus": @(status) }];
+        if (error != NULL) {
+            *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{@"OSStatus":@(status), NSLocalizedDescriptionKey:[NSString stringWithFormat:@"\"%@\" are not supported properties.", supportedProperties]}];
         }
 
         return nil;
@@ -88,9 +93,8 @@ - (instancetype)initWithProtocol:(SDLVideoStreamingProtocol)protocol dimensions:
     NSArray* videoEncoderKeys = self.videoEncoderSettings.allKeys;
     for (NSString *key in videoEncoderKeys) {
         if (CFDictionaryContainsKey(supportedProperties, (__bridge CFStringRef)key) == false) {
-            if (!*error) {
-                NSString *description = [NSString stringWithFormat:@"\"%@\" is not a supported key.", key];
-                *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{NSLocalizedDescriptionKey: description}];
+            if (error != NULL) {
+                *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{NSLocalizedDescriptionKey:[NSString stringWithFormat:@"\"%@\" is not a supported key.", key]}];
             }
             CFRelease(supportedProperties);
             return nil;
@@ -104,9 +108,8 @@ - (instancetype)initWithProtocol:(SDLVideoStreamingProtocol)protocol dimensions:
 
         status = VTSessionSetProperty(self.compressionSession, (__bridge CFStringRef)key, (__bridge CFTypeRef)value);
         if (status != noErr) {
-            if (!*error) {
-                NSString *description = [NSString stringWithFormat:@"Setting key failed \"%@\"", key];
-                *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{NSLocalizedDescriptionKey: description, @"OSStatus": @(status)}];
+            if (error != NULL) {
+                *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{@"OSStatus": @(status), NSLocalizedDescriptionKey:[NSString stringWithFormat:@"Setting key failed \"%@\"", key]}];
             }
             return nil;
         }
@@ -117,8 +120,8 @@ - (instancetype)initWithProtocol:(SDLVideoStreamingProtocol)protocol dimensions:
     } else if ([protocol isEqualToEnum:SDLVideoStreamingProtocolRTP]) {
         _packetizer = [[SDLRTPH264Packetizer alloc] initWithSSRC:ssrc];
     } else {
-        if (!*error) {
-            *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorProtocolUnknown userInfo:@{ @"encoder": protocol}];
+        if (error != NULL) {
+            *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorProtocolUnknown userInfo:@{@"encoder": protocol}];
         }
         return nil;
     }
@@ -129,6 +132,9 @@ - (instancetype)initWithProtocol:(SDLVideoStreamingProtocol)protocol dimensions:
 }
 
 - (void)stop {
+    _currentFrameNumber = 0;
+    _timestampOffset = 0.0;
+
     if (self.compressionSession != NULL) {
         VTCompressionSessionInvalidate(self.compressionSession);
         CFRelease(self.compressionSession);
@@ -176,11 +182,28 @@ - (CVPixelBufferRef CV_NULLABLE)newPixelBuffer {
 }
 
 - (CVPixelBufferPoolRef CV_NULLABLE)pixelBufferPool {
-    return VTCompressionSessionGetPixelBufferPool(self.compressionSession);
+    // HAX: When the app is backgrounded, sometimes the compression session gets invalidated (this can happen the first time the app is backgrounded or the tenth). This causes the pool and/or the compression session to fail when the app is foregrounded and video frames are sent again. Attempt to fix this by recreating the compression session.
+    if (_pixelBufferPool == NULL) {
+        BOOL success = [self sdl_resetCompressionSession];
+        if (!success) {
+            return NULL;
+        }
+
+        _pixelBufferPool = VTCompressionSessionGetPixelBufferPool(self.compressionSession);
+    }
+
+    return _pixelBufferPool;
 }
 
 #pragma mark - Private
 #pragma mark Callback
+
+/// Callback function that VideoToolbox calls when encoding is complete.
+/// @param outputCallbackRefCon The callback's reference value
+/// @param sourceFrameRefCon The frame's reference value
+/// @param status Returns `noErr` if compression was successful, or an error if not successful
+/// @param infoFlags Information about the encode operation (frame dropped or if encode ran asynchronously)
+/// @param sampleBuffer Contains the compressed frame if compression was successful and the frame was not dropped; null otherwise
 void sdl_videoEncoderOutputCallback(void * CM_NULLABLE outputCallbackRefCon, void * CM_NULLABLE sourceFrameRefCon, OSStatus status, VTEncodeInfoFlags infoFlags, CM_NULLABLE CMSampleBufferRef sampleBuffer) {
     // If there was an error in the encoding, drop the frame
     if (status != noErr) {
@@ -312,6 +335,19 @@ + (NSArray *)sdl_extractNalUnitsFromSampleBuffer:(CMSampleBufferRef)sampleBuffer
     return nalUnits;
 }
 
+/// Attempts to create a new VTCompressionSession using the dimensions passed when the video encoder was created and returns whether or not creating the new compression session was created successfully.
+- (BOOL)sdl_resetCompressionSession {
+    // Destroy the current compression session before attempting to create a new one. Otherwise the attempt to create a new compression session sometimes fails.
+    if (self.compressionSession != NULL) {
+        VTCompressionSessionInvalidate(self.compressionSession);
+        CFRelease(self.compressionSession);
+        self.compressionSession = NULL;
+    }
+
+    OSStatus status = VTCompressionSessionCreate(NULL, (int32_t)self.dimensions.width, (int32_t)self.dimensions.height, kCMVideoCodecType_H264, NULL, self.sdl_pixelBufferOptions, NULL, &sdl_videoEncoderOutputCallback, (__bridge void *)self, &_compressionSession);
+    return (status == noErr);
+}
+
 @end
 
 NS_ASSUME_NONNULL_END