Skip to content

Commit

Permalink
fix: safari hls (#48)
Browse files Browse the repository at this point in the history 
* use "software rendering" to fix safari hls
* add an error for hls cors issues
* fix animation frame looping
  • Loading branch information
@brandonocasey
brandonocasey committed Jan 31, 2018
1 parent 4c0c7bb commit 7dc2a69
Show file tree
Hide file tree
Showing 5 changed files with 161 additions and 120 deletions.
199 changes: 94 additions & 105 deletions src/plugin.js
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,18 @@
import {version as VERSION} from '../package.json';
import window from 'global/window';
import document from 'global/document';
import WebVRPolyfill from 'webvr-polyfill';
import videojs from 'video.js';
import * as THREE from 'three';
import VRControls from 'three/examples/js/controls/VRControls.js';
import VREffect from 'three/examples/js/effects/VREffect.js';
import OrbitControls from 'three/examples/js/controls/OrbitControls.js';
import rgbFragmentShader from './rgb-fragment-shader';
import rgbaFragmentShader from './rgba-fragment-shader';
import vertexShader from './vertex-shader';
import * as utils from './utils';

// import controls so they get regisetered with videojs
import './cardboard-button';
import './big-vr-play-button';

const validProjections = [
'360',
'360_LR',
'360_TB',
'360_CUBE',
'NONE',
'AUTO',
'Sphere',
'Cube',
'equirectangular'
];

// Default options for the plugin.
const defaults = {
projection: 'AUTO',
Expand All @@ -43,54 +30,14 @@ const errors = {
headline: '360 not supported on this device',
type: '360_NOT_SUPPORTED',
message: "Your browser does not support 360. See <a href='http://webvr.info'>webvr.info</a> for assistance."
},
'web-vr-hls-cors-not-supported': {
headline: '360 HLS video not supported on this device',
type: '360_NOT_SUPPORTED',
message: "Your browser/device does not support HLS 360 video. See <a href='http://webvr.info'>webvr.info</a> for assistance."
}
};

const getInternalProjectionName = function(projection) {
if (!projection) {
return;
}

projection = projection.toString().trim();

if ((/sphere/i).test(projection)) {
return '360';
}

if ((/cube/i).test(projection)) {
return '360_CUBE';
}

if ((/equirectangular/i).test(projection)) {
return '360';
}

for (let i = 0; i < validProjections.length; i++) {
if (new RegExp('^' + validProjections[i] + '$', 'i').test(projection)) {
return validProjections[i];
}
}

};

const isHLS = function(currentType) {
// hls video types
const hlsTypes = [
// Apple santioned
'application/vnd.apple.mpegurl',
// Very common
'application/x-mpegurl',
// Included for completeness
'video/x-mpegurl',
'video/mpegurl',
'application/mpegurl'
];

// if the current type has a case insensitivie match from the list above
// this is hls
return hlsTypes.some((type) => (new RegExp(`^${type}$`, 'i')).test(currentType));
};

const Plugin = videojs.getPlugin('plugin');
const Component = videojs.getComponent('Component');

Expand All @@ -115,8 +62,9 @@ class VR extends Plugin {
player.errors({errors});
}

// we need this as IE 11 reports that it has a VR display, but isnt compatible with Video as a Texture. for example
if (videojs.browser.IE_VERSION) {
// IE 11 does not support enough webgl to be supported
// older safari does not support cors, so it wont work
if (videojs.browser.IE_VERSION || !utils.corsSupport) {
this.triggerError_({code: 'web-vr-not-supported', dismiss: false});
return;
}
Expand All @@ -132,7 +80,7 @@ class VR extends Plugin {
}

changeProjection_(projection) {
projection = getInternalProjectionName(projection);
projection = utils.getInternalProjectionName(projection);
// don't change to an invalid projection
if (!projection) {
projection = 'NONE';
Expand All @@ -147,7 +95,7 @@ class VR extends Plugin {
// mediainfo cannot be set to auto or we would infinite loop here
// each source should know wether they are 360 or not, if using AUTO
if (this.player_.mediainfo && this.player_.mediainfo.projection && this.player_.mediainfo.projection !== 'AUTO') {
const autoProjection = getInternalProjectionName(this.player_.mediainfo.projection);
const autoProjection = utils.getInternalProjectionName(this.player_.mediainfo.projection);

return this.changeProjection_(autoProjection);
}
Expand Down Expand Up @@ -285,7 +233,15 @@ class VR extends Plugin {
return this.vrDisplay.requestAnimationFrame(fn);
}

return super.requestAnimationFrame(fn);
return Component.prototype.requestAnimationFrame.call(this, fn);
}

cancelAnimationFrame(id) {
if (this.vrDisplay) {
return this.vrDisplay.cancelAnimationFrame(id);
}

return Component.prototype.cancelAnimationFrame.call(this, id);
}

togglePlay_() {
Expand All @@ -297,17 +253,26 @@ class VR extends Plugin {
}

animate_() {
if (!this.initialized_) {
return;
}
if (this.getVideoEl_().readyState === this.getVideoEl_().HAVE_ENOUGH_DATA) {
if (this.videoTexture) {
this.videoTexture.needsUpdate = true;
}
}

// This draws the current video data as an image to a canvas every render. That canvas is used
// as a texture by webgl. Normally the video is used directly and we don't have to do this, but
// HLS video textures on iOS >= 11 is currently broken, so we have to support those browser
// in a roundabout way.
if (this.videoImageContext_) {
this.videoImageContext_.drawImage(this.getVideoEl_(), 0, 0, this.videoImage_.width, this.videoImage_.height);
}

this.controls3d.update();
this.effect.render(this.scene, this.camera);

this.animationFrameId_ = this.requestAnimationFrame(this.animate_);

if (window.navigator.getGamepads) {
// Grab all gamepads
const gamepads = window.navigator.getGamepads();
Expand All @@ -330,21 +295,29 @@ class VR extends Plugin {
}
}
this.camera.getWorldDirection(this.cameraVector);

this.animationFrameId_ = this.requestAnimationFrame(this.animate_);
}

handleResize_() {
const width = this.player_.currentWidth();
const height = this.player_.currentHeight();

// when dealing with a non video
if (this.videoImage_) {
this.videoImage_.width = width;
this.videoImage_.height = height;
}

this.effect.setSize(width, height, false);
this.camera.aspect = width / height;
this.camera.updateProjectionMatrix();
}

setProjection(projection) {

if (!getInternalProjectionName(projection)) {
videojs.log.error('videojs-vr: please pass a valid projection: ' + validProjections.join(', '));
if (!utils.getInternalProjectionName(projection)) {
videojs.log.error('videojs-vr: please pass a valid projection ' + utils.validProjections.join(', '));
return;
}

Expand All @@ -366,46 +339,41 @@ class VR extends Plugin {

this.scene = new THREE.Scene();

this.videoTexture = new THREE.VideoTexture(this.getVideoEl_());

this.videoTexture.generateMipmaps = false;
this.videoTexture.minFilter = THREE.LinearFilter;
this.videoTexture.magFilter = THREE.LinearFilter;

// iOS and macOS HLS fix/hacks
// We opted to stop using a video texture on safari due to
// various bugs that exist when using it. This gives us worse performance
// but it will actually work on all recent version of safari. See
// the following issues for more info on this:
//
// https://bugs.webkit.org/show_bug.cgi?id=163866#c3
// https://github.com/mrdoob/three.js/issues/9754
// On iOS with HLS, color space is wrong and texture is flipped on Y axis
// On macOS, just need to flip texture Y axis

if (isHLS() && videojs.browser.IS_ANY_SAFARI) {
this.log('Safari + iOS + HLS = flipY and colorspace hack');
this.videoTexture.format = THREE.RGBAFormat;
this.videoTexture.flipY = false;
} else if (isHLS() && videojs.browser.IS_SAFARI) {
this.log('Safari + HLS = flipY hack');
this.videoTexture.format = THREE.RGBFormat;
this.videoTexture.flipY = false;
} else {
this.videoTexture.format = THREE.RGBFormat;
}
// https://bugs.webkit.org/show_bug.cgi?id=179417
if (videojs.browser.IS_ANY_SAFARI && utils.isHLS(this.player_.currentSource().type)) {
this.log('Video texture is not supported using image canvas hack');
this.videoImage_ = document.createElement('canvas');
this.videoImage_.width = this.player_.currentWidth();
this.videoImage_.height = this.player_.currentHeight();

if ((this.videoTexture.format === THREE.RGBAFormat || this.videoTexture.format === THREE.RGBFormat) && this.videoTexture.flipY === false) {
let fragmentShader = rgbFragmentShader;
this.videoImageContext_ = this.videoImage_.getContext('2d');
this.videoImageContext_.fillStyle = '#000000';

if (this.videoTexture.format === THREE.RGBAFormat) {
fragmentShader = rgbaFragmentShader;
}
this.videoTexture = new THREE.Texture(this.videoImage_);

this.movieMaterial = new THREE.ShaderMaterial({
uniforms: {texture: {value: this.videoTexture}},
vertexShader,
fragmentShader
});
this.videoTexture.wrapS = THREE.ClampToEdgeWrapping;
this.videoTexture.wrapT = THREE.ClampToEdgeWrapping;
this.videoTexture.flipY = true;
} else {
this.movieMaterial = new THREE.MeshBasicMaterial({ map: this.videoTexture, overdraw: true, side: THREE.DoubleSide });
this.log('Video texture is supported using that');
this.videoTexture = new THREE.VideoTexture(this.getVideoEl_());
}

// shared regardless of wether VideoTexture is used or
// an image canvas is used
this.videoTexture.generateMipmaps = false;
this.videoTexture.minFilter = THREE.LinearFilter;
this.videoTexture.magFilter = THREE.LinearFilter;
this.videoTexture.format = THREE.RGBFormat;

this.movieMaterial = new THREE.MeshBasicMaterial({ map: this.videoTexture, overdraw: true, side: THREE.DoubleSide });

this.changeProjection_(this.currentProjection_);

if (this.currentProjection_ === 'NONE') {
Expand Down Expand Up @@ -438,6 +406,21 @@ class VR extends Plugin {
antialias: true
});

const webglContext = this.renderer.getContext('webgl');
const oldTexImage2D = webglContext.texImage2D;

/* this is a workaround since threejs uses try catch */
webglContext.texImage2D = (...args) => {
try {
return oldTexImage2D.apply(webglContext, args);
} catch (e) {
this.reset();
this.player_.pause();
this.triggerError_({code: 'web-vr-hls-cors-not-supported', dismiss: false});
throw new Error(e);
}
};

this.renderer.setSize(this.player_.currentWidth(), this.player_.currentHeight(), false);
this.effect = new VREffect(this.renderer);

Expand Down Expand Up @@ -501,7 +484,7 @@ class VR extends Plugin {
this.controls3d = new OrbitControls(this.camera, this.renderedCanvas);
this.controls3d.target.set(0, 0, -1);
}
this.requestAnimationFrame(this.animate_);
this.animationFrameId_ = this.requestAnimationFrame(this.animate_);
});
} else if (window.navigator.getVRDevices) {
this.triggerError_({code: 'web-vr-out-of-date', dismiss: false});
Expand Down Expand Up @@ -583,6 +566,14 @@ class VR extends Plugin {
this.cancelAnimationFrame(this.animationFrameId_);
}

if (this.videoImage_) {
this.videoImage_ = null;
}

if (this.videoImageContext_) {
this.videoImageContext_ = null;
}

this.initialized_ = false;
}

Expand All @@ -592,8 +583,6 @@ class VR extends Plugin {
}
}

VR.prototype.requestAnimationFrame = Component.prototype.requestAnimationFrame;
VR.prototype.cancelAnimationFrame = Component.prototype.cancelAnimationFrame;
VR.prototype.setTimeout = Component.prototype.setTimeout;
VR.prototype.clearTimeout = Component.prototype.clearTimeout;

Expand Down
5 changes: 0 additions & 5 deletions src/rgb-fragment-shader.js
View file

This file was deleted.

5 changes: 0 additions & 5 deletions src/rgba-fragment-shader.js
View file

This file was deleted.

Loading

0 comments on commit 7dc2a69

Please sign in to comment.