From dccb28da887ba6a51302a34a819d82c81387c850 Mon Sep 17 00:00:00 2001 From: Julian Domingo Date: Thu, 1 Dec 2022 09:18:24 -0800 Subject: [PATCH] fix(cast): Use cast platform APIs in MediaCapabilties polyfill (#4727) See https://github.com/shaka-project/shaka-player/issues/4726 for more context. This allows Cast devices to properly filter stream variants with a resolution surpassing that of the device's capabilities. We place the fix in the `MediaCapabilities` polyfill since it's intended to be the right way to check for anything related to platform support. HDR support checks will require `eotf=smpte2048`, as indicated in https://github.com/shaka-project/shaka-player/issues/2813#issue-684874730. Specifically, a `{hev|hvc}1.2` profile is only an *indication* of an HDR transfer function, but *may* be a non-HDR 10-bit color stream. In Cast, the platform can distinguish between the two by explicitly providing the transfer function; it uses `smpte2048` (`"PQ"`) because this is the "basis of HDR video formats..." (https://en.wikipedia.org/wiki/Perceptual_quantizer). --- lib/polyfill/media_capabilities.js | 68 +++++- test/polyfill/media_capabilities_unit.js | 261 +++++++++++++++++++++++ 2 files changed, 320 insertions(+), 9 deletions(-) create mode 100644 test/polyfill/media_capabilities_unit.js diff --git a/lib/polyfill/media_capabilities.js b/lib/polyfill/media_capabilities.js index 825470b358..d4d608839d 100644 --- a/lib/polyfill/media_capabilities.js +++ b/lib/polyfill/media_capabilities.js @@ -8,6 +8,7 @@ goog.provide('shaka.polyfill.MediaCapabilities'); goog.require('shaka.log'); goog.require('shaka.polyfill'); +goog.require('shaka.util.Error'); goog.require('shaka.util.Platform'); @@ -84,37 +85,48 @@ shaka.polyfill.MediaCapabilities = class { return res; } + const videoConfig = mediaDecodingConfig['video']; + const audioConfig = mediaDecodingConfig['audio']; + if (mediaDecodingConfig.type == 'media-source') { if (!shaka.util.Platform.supportsMediaSource()) { return res; } // Use 'MediaSource.isTypeSupported' to check if the stream is supported. - if (mediaDecodingConfig['video']) { - const contentType = mediaDecodingConfig['video'].contentType; - const isSupported = MediaSource.isTypeSupported(contentType); + // Cast platforms will additionally check canDisplayType(), which + // accepts extended MIME type parameters. + // See: https://github.com/shaka-project/shaka-player/issues/4726 + if (videoConfig) { + let isSupported; + if (shaka.util.Platform.isChromecast()) { + isSupported = + shaka.polyfill.MediaCapabilities.canCastDisplayType_(videoConfig); + } else { + isSupported = MediaSource.isTypeSupported(videoConfig.contentType); + } if (!isSupported) { return res; } } - if (mediaDecodingConfig['audio']) { - const contentType = mediaDecodingConfig['audio'].contentType; + if (audioConfig) { + const contentType = audioConfig.contentType; const isSupported = MediaSource.isTypeSupported(contentType); if (!isSupported) { return res; } } } else if (mediaDecodingConfig.type == 'file') { - if (mediaDecodingConfig['video']) { - const contentType = mediaDecodingConfig['video'].contentType; + if (videoConfig) { + const contentType = videoConfig.contentType; const isSupported = shaka.util.Platform.supportsMediaType(contentType); if (!isSupported) { return res; } } - if (mediaDecodingConfig['audio']) { - const contentType = mediaDecodingConfig['audio'].contentType; + if (audioConfig) { + const contentType = audioConfig.contentType; const isSupported = shaka.util.Platform.supportsMediaType(contentType); if (!isSupported) { return res; @@ -189,6 +201,44 @@ shaka.polyfill.MediaCapabilities = class { return res; } + + /** + * Checks if the given media parameters of the video or audio streams are + * supported by the Cast platform. + * @param {!VideoConfiguration} videoConfig The 'video' field of the + * MediaDecodingConfiguration. + * @return {boolean} `true` when the stream can be displayed on a Cast device. + * @private + */ + static canCastDisplayType_(videoConfig) { + if (!(window.cast)) { + shaka.log.error('Expected cast namespace to be available!'); + throw new shaka.util.Error( + shaka.util.Error.Severity.CRITICAL, + shaka.util.Error.Category.CAST, + shaka.util.Error.Code.CAST_API_UNAVAILABLE); + } else if (!(cast.__platform__ && cast.__platform__.canDisplayType)) { + shaka.log.warning('Expected cast APIs to be available! Falling back to ' + + 'MediaSource.isTypeSupported() for type support.'); + return MediaSource.isTypeSupported(videoConfig.contentType); + } + + let displayType = videoConfig.contentType; + if (videoConfig.width && videoConfig.height) { + displayType += + `; width=${videoConfig.width}; height=${videoConfig.height}`; + } + if (videoConfig.framerate) { + displayType += `; framerate=${videoConfig.framerate}`; + } + if (videoConfig.transferFunction === 'pq') { + // A "PQ" transfer function indicates this is an HDR-capable stream; + // "smpte2084" is the published standard. We need to inform the platform + // this query is specifically for HDR. + displayType += '; eotf=smpte2084'; + } + return cast.__platform__.canDisplayType(displayType); + } }; /** diff --git a/test/polyfill/media_capabilities_unit.js b/test/polyfill/media_capabilities_unit.js new file mode 100644 index 0000000000..01afd990ae --- /dev/null +++ b/test/polyfill/media_capabilities_unit.js @@ -0,0 +1,261 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +describe('MediaCapabilities', () => { + const Util = shaka.test.Util; + const originalCast = window['cast']; + const originalVendor = navigator.vendor; + const originalUserAgent = navigator.userAgent; + const originalRequestMediaKeySystemAccess = + navigator.requestMediaKeySystemAccess; + const originalMediaCapabilities = navigator.mediaCapabilities; + + /** @type {MediaDecodingConfiguration} */ + let mockDecodingConfig; + /** @type {!jasmine.Spy} */ + let mockCanDisplayType; + + beforeAll(() => { + Object.defineProperty(window['navigator'], + 'userAgent', { + value: 'unknown', configurable: true, + writable: true, + }); + Object.defineProperty(window['navigator'], + 'vendor', { + value: 'unknown', configurable: true, + writable: true, + }); + Object.defineProperty(window['navigator'], + 'requestMediaKeySystemAccess', { + value: 'unknown', configurable: true, + writable: true, + }); + Object.defineProperty(window['navigator'], + 'mediaCapabilities', { + value: undefined, configurable: true, + writable: true, + }); + }); + + beforeEach(() => { + mockDecodingConfig = { + audio: { + bitrate: 100891, + channels: 2, + contentType: 'audio/mp4; codecs="mp4a.40.2"', + samplerate: 48000, + spatialRendering: false, + }, + keySystemConfiguration: { + audio: {robustness: 'SW_SECURE_CRYPTO'}, + distinctiveIdentifier: 'optional', + initDataType: 'cenc', + keySystem: 'com.widevine.alpha', + persistentState: 'optional', + sessionTypes: ['temporary'], + video: {robustness: 'SW_SECURE_CRYPTO'}, + }, + type: 'media-source', + video: { + bitrate: 349265, + contentType: 'video/mp4; codecs="avc1.4D4015"', + framerate: 23.976023976023978, + height: 288, + width: 512, + }, + }; + shaka.polyfill.MediaCapabilities.memoizedMediaKeySystemAccessRequests_ = {}; + + mockCanDisplayType = jasmine.createSpy('canDisplayType'); + mockCanDisplayType.and.returnValue(false); + }); + + afterEach(() => { + window['cast'] = originalCast; + }); + + afterAll(() => { + window['cast'] = originalCast; + Object.defineProperty(window['navigator'], + 'userAgent', {value: originalUserAgent}); + Object.defineProperty(window['navigator'], + 'vendor', {value: originalVendor}); + Object.defineProperty(window['navigator'], + 'requestMediaKeySystemAccess', + {value: originalRequestMediaKeySystemAccess}); + Object.defineProperty(window['navigator'], + 'mediaCapabilities', {value: originalMediaCapabilities}); + }); + + describe('install', () => { + it('should define decoding info method', () => { + shaka.polyfill.MediaCapabilities.install(); + + expect(navigator.mediaCapabilities.decodingInfo).toBeDefined(); + }); + }); + + describe('decodingInfo', () => { + it('should check codec support when MediaDecodingConfiguration.type ' + + 'is "media-source"', () => { + const isTypeSupportedSpy = + spyOn(window['MediaSource'], 'isTypeSupported').and.returnValue(true); + shaka.polyfill.MediaCapabilities.install(); + navigator.mediaCapabilities.decodingInfo(mockDecodingConfig); + + expect(isTypeSupportedSpy).toHaveBeenCalledTimes(2); + expect(isTypeSupportedSpy).toHaveBeenCalledWith( + mockDecodingConfig.video.contentType, + ); + expect(isTypeSupportedSpy).toHaveBeenCalledWith( + mockDecodingConfig.audio.contentType, + ); + }); + + it('should check codec support when MediaDecodingConfiguration.type ' + + 'is "file"', () => { + const supportsMediaTypeSpy = + spyOn(shaka['util']['Platform'], + 'supportsMediaType').and.returnValue(true); + mockDecodingConfig.type = 'file'; + shaka.polyfill.MediaCapabilities.install(); + navigator.mediaCapabilities.decodingInfo(mockDecodingConfig); + + expect(supportsMediaTypeSpy).toHaveBeenCalledTimes(2); + expect(supportsMediaTypeSpy).toHaveBeenCalledWith( + mockDecodingConfig.video.contentType, + ); + expect(supportsMediaTypeSpy).toHaveBeenCalledWith( + mockDecodingConfig.audio.contentType, + ); + }); + + it('should check MediaKeySystem when keySystemConfiguration is present', + async () => { + const mockResult = {mockKeySystemAccess: 'mockKeySystemAccess'}; + spyOn(window['MediaSource'], 'isTypeSupported').and.returnValue(true); + const requestKeySystemAccessSpy = + spyOn(window['navigator'], + 'requestMediaKeySystemAccess').and.returnValue(mockResult); + + shaka.polyfill.MediaCapabilities.install(); + const result = await navigator.mediaCapabilities + .decodingInfo(mockDecodingConfig); + + expect(requestKeySystemAccessSpy).toHaveBeenCalledWith( + 'com.widevine.alpha', + [{ + audioCapabilities: [ + { + robustness: 'SW_SECURE_CRYPTO', + contentType: 'audio/mp4; codecs="mp4a.40.2"', + }, + ], + distinctiveIdentifier: 'optional', + initDataTypes: ['cenc'], + persistentState: 'optional', + sessionTypes: ['temporary'], + videoCapabilities: [{ + robustness: 'SW_SECURE_CRYPTO', + contentType: 'video/mp4; codecs="avc1.4D4015"', + }], + }], + ); + expect(result.keySystemAccess).toEqual(mockResult); + }); + + it('throws when the cast namespace is not available', async () => { + // Temporarily remove window.cast to trigger error. It's restored after + // every test. + delete window['cast']; + + const isChromecastSpy = + spyOn(shaka['util']['Platform'], + 'isChromecast').and.returnValue(true); + const expected = Util.jasmineError(new shaka.util.Error( + shaka.util.Error.Severity.CRITICAL, + shaka.util.Error.Category.CAST, + shaka.util.Error.Code.CAST_API_UNAVAILABLE)); + const isTypeSupportedSpy = + spyOn(window['MediaSource'], 'isTypeSupported').and.returnValue(true); + + shaka.polyfill.MediaCapabilities.install(); + await expectAsync( + navigator.mediaCapabilities.decodingInfo(mockDecodingConfig)) + .toBeRejectedWith(expected); + + expect(isTypeSupportedSpy).not.toHaveBeenCalled(); + // 1 (during install()) + 1 (for video config check). + expect(isChromecastSpy).toHaveBeenCalledTimes(2); + }); + + it('falls back to isTypeSupported() when canDisplayType() missing', + async () => { + // We only set the cast namespace, but not the canDisplayType() API. + window['cast'] = {}; + const isChromecastSpy = + spyOn(shaka['util']['Platform'], + 'isChromecast').and.returnValue(true); + const isTypeSupportedSpy = + spyOn(window['MediaSource'], 'isTypeSupported') + .and.returnValue(true); + + shaka.polyfill.MediaCapabilities.install(); + await navigator.mediaCapabilities.decodingInfo(mockDecodingConfig); + + expect(mockCanDisplayType).not.toHaveBeenCalled(); + // 1 (during install()) + 1 (for video config check). + expect(isChromecastSpy).toHaveBeenCalledTimes(2); + // 1 (fallback in canCastDisplayType()) + + // 1 (mockDecodingConfig.audio). + expect(isTypeSupportedSpy).toHaveBeenCalledTimes(2); + }); + + it('should use cast.__platform__.canDisplayType for "supported" field ' + + 'when platform is Cast', async () => { + // We're using quotes to access window.cast because the compiler + // knows about lots of Cast-specific APIs we aren't mocking. We + // don't need this mock strictly type-checked. + window['cast'] = { + __platform__: {canDisplayType: mockCanDisplayType}, + }; + const isChromecastSpy = + spyOn(shaka['util']['Platform'], + 'isChromecast').and.returnValue(true); + const isTypeSupportedSpy = + spyOn(window['MediaSource'], 'isTypeSupported').and.returnValue(true); + + // Tests an HDR stream's extended MIME type is correctly provided. + mockDecodingConfig.video.transferFunction = 'pq'; + mockDecodingConfig.video.contentType = + 'video/mp4; codecs="hev1.2.4.L153.B0"'; + // Round to a whole number since we can't rely on number => string + // conversion precision on all devices. + mockDecodingConfig.video.framerate = 24; + mockCanDisplayType.and.callFake((type) => { + expect(type).toBe( + 'video/mp4; ' + + 'codecs="hev1.2.4.L153.B0"; ' + + 'width=512; ' + + 'height=288; ' + + 'framerate=24; ' + + 'eotf=smpte2084'); + return true; + }); + + shaka.polyfill.MediaCapabilities.install(); + await navigator.mediaCapabilities.decodingInfo(mockDecodingConfig); + + // 1 (during install()) + 1 (for video config check). + expect(isChromecastSpy).toHaveBeenCalledTimes(2); + // 1 (mockDecodingConfig.audio). + expect(isTypeSupportedSpy).toHaveBeenCalledTimes(1); + // Called once in canCastDisplayType. + expect(mockCanDisplayType).toHaveBeenCalledTimes(1); + }); + }); +});