Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf: Caching mediaSource support for browser engine #4778

Merged
merged 5 commits into from
Dec 6, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AUTHORS
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,4 @@ Raymond Cheng <[email protected]>
Blue Billywig <*@bluebillywig.com>
João Nabais <[email protected]>
Koen Romers <[email protected]>
Zhenghang Chen <[email protected]>
1 change: 1 addition & 0 deletions CONTRIBUTORS
Original file line number Diff line number Diff line change
Expand Up @@ -134,3 +134,4 @@ Raymond Cheng <[email protected]>
Janroel Koppen <[email protected]>
João Nabais <[email protected]>
Koen Romers <[email protected]>
Zhenghang Chen <[email protected]>
1 change: 1 addition & 0 deletions build/types/core
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
+../../lib/media/drm_engine.js
+../../lib/media/gap_jumping_controller.js
+../../lib/media/manifest_parser.js
+../../lib/media/media_source_capabilities.js
+../../lib/media/media_source_engine.js
+../../lib/media/mp4_segment_index_parser.js
+../../lib/media/play_rate_controller.js
Expand Down
36 changes: 36 additions & 0 deletions lib/media/media_source_capabilities.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*! @license
* Shaka Player
* Copyright 2016 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/

goog.provide('shaka.media.Capabilities');

/**
* @summary
* This is for capturing all media source capabilities on current platform.
* And this is for static check and can not be constructed.
*/
shaka.media.Capabilities = class {
/**
* Cache browser engine call to improve performance on some poor platforms
*
* @param {string} type
* @return {boolean}
*/
static isTypeSupported(type) {
const supportMap = shaka.media.Capabilities.MediaSourceTypeSupportMap;
if (supportMap.has(type)) {
return supportMap.get(type);
}
const currentSupport = MediaSource.isTypeSupported(type);
supportMap.set(type, currentSupport);
return currentSupport;
}
};

/**
* Public it for unit test, and developer could also check the support map.
* @type {!Map.<string, boolean>}
*/
shaka.media.Capabilities.MediaSourceTypeSupportMap = new Map();
8 changes: 5 additions & 3 deletions lib/media/media_source_engine.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ goog.provide('shaka.media.MediaSourceEngine');

goog.require('goog.asserts');
goog.require('shaka.log');
goog.require('shaka.media.Capabilities');
goog.require('shaka.media.ContentWorkarounds');
goog.require('shaka.media.ClosedCaptionParser');
goog.require('shaka.media.IClosedCaptionParser');
Expand Down Expand Up @@ -175,7 +176,7 @@ shaka.media.MediaSourceEngine = class {
stream.mimeType, stream.codecs);
const extendedMimeType = shaka.util.MimeUtils.getExtendedType(stream);
return shaka.text.TextEngine.isTypeSupported(fullMimeType) ||
MediaSource.isTypeSupported(extendedMimeType) ||
shaka.media.Capabilities.isTypeSupported(extendedMimeType) ||
shaka.media.Transmuxer.isSupported(fullMimeType, stream.type);
}

Expand Down Expand Up @@ -231,7 +232,7 @@ shaka.media.MediaSourceEngine = class {
if (shaka.text.TextEngine.isTypeSupported(type)) {
support[type] = true;
} else {
support[type] = MediaSource.isTypeSupported(type) ||
support[type] = shaka.media.Capabilities.isTypeSupported(type) ||
shaka.media.Transmuxer.isSupported(type);
}
} else {
Expand Down Expand Up @@ -365,7 +366,8 @@ shaka.media.MediaSourceEngine = class {
if (contentType == ContentType.TEXT) {
this.reinitText(mimeType, sequenceMode);
} else {
if ((forceTransmux || !MediaSource.isTypeSupported(mimeType)) &&
if ((forceTransmux ||
!shaka.media.Capabilities.isTypeSupported(mimeType)) &&
shaka.media.Transmuxer.isSupported(mimeType, contentType)) {
this.transmuxers_[contentType] =
new shaka.media.Transmuxer(mimeType);
Expand Down
10 changes: 6 additions & 4 deletions lib/media/transmuxer.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
goog.provide('shaka.media.Transmuxer');

goog.require('goog.asserts');
goog.require('shaka.media.Capabilities');
goog.require('shaka.util.BufferUtils');
goog.require('shaka.util.Error');
goog.require('shaka.util.IDestroyable');
Expand Down Expand Up @@ -71,6 +72,7 @@ shaka.media.Transmuxer = class {
*/
static isSupported(mimeType, contentType) {
const Transmuxer = shaka.media.Transmuxer;
const Capabilities = shaka.media.Capabilities;

const isTs = Transmuxer.isTsContainer_(mimeType);
const isAac = Transmuxer.isAacContainer_(mimeType);
Expand All @@ -80,20 +82,20 @@ shaka.media.Transmuxer = class {
}

if (isAac) {
return MediaSource.isTypeSupported(Transmuxer.convertAacCodecs_());
return Capabilities.isTypeSupported(Transmuxer.convertAacCodecs_());
}

if (contentType) {
return MediaSource.isTypeSupported(
return Capabilities.isTypeSupported(
Transmuxer.convertTsCodecs_(contentType, mimeType));
}

const ContentType = shaka.util.ManifestParserUtils.ContentType;

const audioMime = Transmuxer.convertTsCodecs_(ContentType.AUDIO, mimeType);
const videoMime = Transmuxer.convertTsCodecs_(ContentType.VIDEO, mimeType);
return MediaSource.isTypeSupported(audioMime) ||
MediaSource.isTypeSupported(videoMime);
return Capabilities.isTypeSupported(audioMime) ||
Capabilities.isTypeSupported(videoMime);
}


Expand Down
14 changes: 9 additions & 5 deletions lib/polyfill/media_capabilities.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
goog.provide('shaka.polyfill.MediaCapabilities');

goog.require('shaka.log');
goog.require('shaka.media.Capabilities');
goog.require('shaka.polyfill');
goog.require('shaka.util.Error');
goog.require('shaka.util.Platform');
Expand Down Expand Up @@ -100,11 +101,14 @@ shaka.polyfill.MediaCapabilities = class {
const videoConfig = mediaDecodingConfig['video'];
const audioConfig = mediaDecodingConfig['audio'];

const Capabilities = shaka.media.Capabilities;

if (mediaDecodingConfig.type == 'media-source') {
if (!shaka.util.Platform.supportsMediaSource()) {
return res;
}
// Use 'MediaSource.isTypeSupported' to check if the stream is supported.
// Use 'shaka.media.Capabilities.isTypeSupported'to check if
// the stream is supported.
// Cast platforms will additionally check canDisplayType(), which
// accepts extended MIME type parameters.
// See: https://github.com/shaka-project/shaka-player/issues/4726
Expand All @@ -114,7 +118,7 @@ shaka.polyfill.MediaCapabilities = class {
isSupported =
shaka.polyfill.MediaCapabilities.canCastDisplayType_(videoConfig);
} else {
isSupported = MediaSource.isTypeSupported(videoConfig.contentType);
isSupported = Capabilities.isTypeSupported(videoConfig.contentType);
}
if (!isSupported) {
return res;
Expand All @@ -123,7 +127,7 @@ shaka.polyfill.MediaCapabilities = class {

if (audioConfig) {
const contentType = audioConfig.contentType;
const isSupported = MediaSource.isTypeSupported(contentType);
const isSupported = Capabilities.isTypeSupported(contentType);
if (!isSupported) {
return res;
}
Expand Down Expand Up @@ -246,8 +250,8 @@ shaka.polyfill.MediaCapabilities = class {
shaka.util.Error.Code.CAST_API_UNAVAILABLE);
} else if (!(cast.__platform__ && cast.__platform__.canDisplayType)) {
shaka.log.warning('Expected cast APIs to be available! Falling back to ' +
'MediaSource.isTypeSupported() for type support.');
return MediaSource.isTypeSupported(videoConfig.contentType);
'shaka.media.Capabilities.isTypeSupported() for type support.');
return shaka.media.Capabilities.isTypeSupported(videoConfig.contentType);
}

let displayType = videoConfig.contentType;
Expand Down
8 changes: 5 additions & 3 deletions lib/util/stream_utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ goog.provide('shaka.util.StreamUtils');

goog.require('goog.asserts');
goog.require('shaka.log');
goog.require('shaka.media.Capabilities');
goog.require('shaka.text.TextEngine');
goog.require('shaka.util.Functional');
goog.require('shaka.util.LanguageUtils');
Expand Down Expand Up @@ -437,6 +438,7 @@ shaka.util.StreamUtils = class {
// See: https://github.com/shaka-project/shaka-player/issues/3860
const video = variant.video;
const ContentType = shaka.util.ManifestParserUtils.ContentType;
const Capabilities = shaka.media.Capabilities;
if (video) {
let videoCodecs =
shaka.util.StreamUtils.getCorrectVideoCodecs_(video.codecs);
Expand All @@ -454,13 +456,13 @@ shaka.util.StreamUtils = class {
shaka.util.StreamUtils.getCorrectAudioCodecs_(audioCodecs);
const audioFullType = shaka.util.MimeUtils.getFullOrConvertedType(
video.mimeType, audioCodecs, ContentType.AUDIO);
if (!MediaSource.isTypeSupported(audioFullType)) {
if (!Capabilities.isTypeSupported(audioFullType)) {
return false;
}
}
const fullType = shaka.util.MimeUtils.getFullOrConvertedType(
video.mimeType, videoCodecs, ContentType.VIDEO);
if (!MediaSource.isTypeSupported(fullType)) {
if (!Capabilities.isTypeSupported(fullType)) {
return false;
}
}
Expand All @@ -470,7 +472,7 @@ shaka.util.StreamUtils = class {
shaka.util.StreamUtils.getCorrectAudioCodecs_(audio.codecs);
const fullType = shaka.util.MimeUtils.getFullOrConvertedType(
audio.mimeType, codecs, ContentType.AUDIO);
if (!MediaSource.isTypeSupported(fullType)) {
if (!Capabilities.isTypeSupported(fullType)) {
return false;
}
}
Expand Down
44 changes: 21 additions & 23 deletions test/polyfill/media_capabilities_unit.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ describe('MediaCapabilities', () => {
const originalRequestMediaKeySystemAccess =
navigator.requestMediaKeySystemAccess;
const originalMediaCapabilities = navigator.mediaCapabilities;
const supportMap = shaka.media.Capabilities.MediaSourceTypeSupportMap;

/** @type {MediaDecodingConfiguration} */
let mockDecodingConfig;
Expand Down Expand Up @@ -102,18 +103,12 @@ describe('MediaCapabilities', () => {
describe('decodingInfo', () => {
it('should check codec support when MediaDecodingConfiguration.type ' +
'is "media-source"', () => {
const isTypeSupportedSpy =
spyOn(window['MediaSource'], 'isTypeSupported').and.returnValue(true);
expect(window['MediaSource']['isTypeSupported']).toBeDefined();
shaka.polyfill.MediaCapabilities.install();
navigator.mediaCapabilities.decodingInfo(mockDecodingConfig);

expect(isTypeSupportedSpy).toHaveBeenCalledTimes(2);
expect(isTypeSupportedSpy).toHaveBeenCalledWith(
mockDecodingConfig.video.contentType,
);
expect(isTypeSupportedSpy).toHaveBeenCalledWith(
mockDecodingConfig.audio.contentType,
);
expect(supportMap.has(mockDecodingConfig.video.contentType)).toBe(true);
expect(supportMap.has(mockDecodingConfig.audio.contentType)).toBe(true);
});

it('should check codec support when MediaDecodingConfiguration.type ' +
Expand Down Expand Up @@ -218,9 +213,7 @@ describe('MediaCapabilities', () => {
const isChromecastSpy =
spyOn(shaka['util']['Platform'],
'isChromecast').and.returnValue(true);
const isTypeSupportedSpy =
spyOn(window['MediaSource'], 'isTypeSupported')
.and.returnValue(true);
expect(window['MediaSource']['isTypeSupported']).toBeDefined();

shaka.polyfill.MediaCapabilities.install();
await navigator.mediaCapabilities.decodingInfo(mockDecodingConfig);
Expand All @@ -230,7 +223,10 @@ describe('MediaCapabilities', () => {
expect(isChromecastSpy).toHaveBeenCalledTimes(2);
// 1 (fallback in canCastDisplayType()) +
// 1 (mockDecodingConfig.audio).
expect(isTypeSupportedSpy).toHaveBeenCalledTimes(2);
expect(supportMap.has(mockDecodingConfig.video.contentType))
.toBe(true);
expect(supportMap.has(mockDecodingConfig.audio.contentType))
.toBe(true);
});

it('should use cast.__platform__.canDisplayType for "supported" field ' +
Expand All @@ -244,8 +240,7 @@ describe('MediaCapabilities', () => {
const isChromecastSpy =
spyOn(shaka['util']['Platform'],
'isChromecast').and.returnValue(true);
const isTypeSupportedSpy =
spyOn(window['MediaSource'], 'isTypeSupported').and.returnValue(true);
expect(window['MediaSource']['isTypeSupported']).toBeDefined();

// Tests an HDR stream's extended MIME type is correctly provided.
mockDecodingConfig.video.transferFunction = 'pq';
Expand All @@ -254,14 +249,17 @@ describe('MediaCapabilities', () => {
// Round to a whole number since we can't rely on number => string
// conversion precision on all devices.
mockDecodingConfig.video.framerate = 24;

const chromecastType =
'video/mp4; ' +
'codecs="hev1.2.4.L153.B0"; ' +
'width=512; ' +
'height=288; ' +
'framerate=24; ' +
'eotf=smpte2084';
mockCanDisplayType.and.callFake((type) => {
expect(type).toBe(
'video/mp4; ' +
'codecs="hev1.2.4.L153.B0"; ' +
'width=512; ' +
'height=288; ' +
'framerate=24; ' +
'eotf=smpte2084');
expect(type).toBe(chromecastType);
supportMap.set(type, true);
return true;
});

Expand All @@ -271,7 +269,7 @@ describe('MediaCapabilities', () => {
// 1 (during install()) + 1 (for video config check).
expect(isChromecastSpy).toHaveBeenCalledTimes(2);
// 1 (mockDecodingConfig.audio).
expect(isTypeSupportedSpy).toHaveBeenCalledTimes(1);
expect(supportMap.has(chromecastType)).toBe(true);
// Called once in canCastDisplayType.
expect(mockCanDisplayType).toHaveBeenCalledTimes(1);
});
Expand Down